app-import/src/main/java/eu/etaxonomy/cdm/app/berlinModelImport/EuroMedActivator.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.app.berlinModelImport;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.io.InputStreamReader;
  15 import java.io.Reader;
  16 import java.util.Arrays;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Set;
  22 import java.util.UUID;
  23 import java.util.stream.Collectors;
  24
  25 import org.apache.log4j.Logger;
  26 import org.springframework.security.core.GrantedAuthority;
  27 import org.springframework.transaction.TransactionStatus;
  28
  29 import eu.etaxonomy.cdm.api.application.FirstDataInserter;
  30 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  31 import eu.etaxonomy.cdm.api.service.IGroupService;
  32 import eu.etaxonomy.cdm.api.service.description.AggregationMode;
  33 import eu.etaxonomy.cdm.api.service.description.DistributionAggregation;
  34 import eu.etaxonomy.cdm.api.service.description.DistributionAggregationConfiguration;
  35 import eu.etaxonomy.cdm.api.service.pager.Pager;
  36 import eu.etaxonomy.cdm.app.common.CdmDestinations;
  37 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  38 import eu.etaxonomy.cdm.database.DbSchemaValidation;
  39 import eu.etaxonomy.cdm.database.ICdmDataSource;
  40 import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
  41 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  42 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
  43 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
  44 import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
  45 import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
  46 import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;
  47 import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
  48 import eu.etaxonomy.cdm.io.common.Source;
  49 import eu.etaxonomy.cdm.model.agent.Person;
  50 import eu.etaxonomy.cdm.model.common.Language;
  51 import eu.etaxonomy.cdm.model.description.Feature;
  52 import eu.etaxonomy.cdm.model.location.NamedArea;
  53 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  54 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  55 import eu.etaxonomy.cdm.model.metadata.CdmPreference;
  56 import eu.etaxonomy.cdm.model.metadata.PreferencePredicate;
  57 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  58 import eu.etaxonomy.cdm.model.name.Rank;
  59 import eu.etaxonomy.cdm.model.permission.GrantedAuthorityImpl;
  60 import eu.etaxonomy.cdm.model.permission.Group;
  61 import eu.etaxonomy.cdm.model.permission.User;
  62 import eu.etaxonomy.cdm.model.taxon.Taxon;
  63 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  64 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  65 import eu.etaxonomy.cdm.model.term.Representation;
  66 import eu.etaxonomy.cdm.model.term.TermTree;
  67 import eu.etaxonomy.cdm.persistence.permission.Role;
  68 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  69 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  70
  71
  72 /**
  73  * TODO add the following to a wiki page:
  74  * HINT: If you are about to import into a mysql data base running under windows and if you wish to
  75  * dump and restore the resulting data bas under another operation systen
  76  * you must set the mysql system variable lower_case_table_names = 0 in order to create data base with table compatible names.
  77  *
  78  * @author a.mueller
  79  */
  80 public class EuroMedActivator {
  81         private static final Logger logger = Logger.getLogger(EuroMedActivator.class);
  82
  83         //database validation status (create, update, validate ...)
  84         static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
  85 //  static final Source berlinModelSource = BerlinModelSources.euroMed_Pub2();
  86         static final Source berlinModelSource = BerlinModelSources.euroMed_BGBM42();
  87 //      static final Source berlinModelSource = BerlinModelSources.euroMed_PESI3();
  88 //
  89     static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
  90 //  static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed();
  91 //    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed2();
  92 //      static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_euroMed();
  93 //  static final ICdmDataSource cdmDestination = CdmDestinations.cdm_production_euromed();
  94
  95     //check - import
  96     static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
  97
  98     static final boolean doUser = true;
  99 //  //authors
 100     static final boolean doAuthors = true;
 101     //references
 102     static final DO_REFERENCES doReferences = DO_REFERENCES.ALL;
 103     //names
 104     static final boolean doTaxonNames = true;
 105     static final boolean doRelNames = true;
 106     static final boolean doNameStatus = true;
 107     static final boolean doNameFacts = true;
 108
 109     //taxa
 110     static final boolean doTaxa = true;
 111     static final boolean doFacts = true;
 112     static final boolean doRelTaxa = true;
 113     static final boolean doOccurrences = true;
 114     static final boolean doOccurrenceSources = true;
 115     static final boolean doCommonNames = true;
 116
 117     static final boolean doNamedAreas = true;
 118
 119   //serious types do not exist in E+M except for name types which are handled in name relations
 120     static final boolean doTypes = false;  //serious types do not exist in E+M except for name types which are handled in name relations
 121
 122
 123     static final boolean doRunTransmissionEngine = false; // (hbm2dll == DbSchemaValidation.VALIDATE);
 124
 125     //etc.
 126     static final boolean doMarker = false;  //no relevant markers exist
 127
 128     boolean invers = !(hbm2dll == DbSchemaValidation.CREATE);
 129
 130     boolean doPreliminaryRefDetailsWithNames = true;
 131
 132     boolean logNotMatchingOldNames = false;
 133     boolean logMatchingNotExportedOldNames = false;  //true
 134     boolean checkOldNameIsSynonym = false;
 135     boolean includeMANsForOldNameCheck = true;
 136
 137         static final boolean includePesiExport = false;
 138
 139         static final int sourceSecId = 7000000; //500000
 140         static final UUID classificationUuid = UUID.fromString("314a68f9-8449-495a-91c2-92fde8bcf344");
 141         static final boolean useSingleClassification = true;
 142         static final String classificationName = "Euro+Med 2018";
 143         static final UUID featureTreeUuid = UUID.fromString("6a5e1c2b-ec0d-46c8-9c7d-a2059267ffb7");
 144         static final Object[] featureKeyList = new Integer[]{1, 31, 4, 98, 41};
 145
 146         // set to zero for unlimited nameFacts
 147         static final int maximumNumberOfNameFacts = 0;
 148
 149         static final int partitionSize = 2500;
 150
 151
 152         //editor - import
 153         static final EDITOR editor = EDITOR.EDITOR_AS_EDITOR;
 154
 155         //NomenclaturalCode
 156         static final NomenclaturalCode nomenclaturalCode = NomenclaturalCode.ICNAFP;
 157
 158         //ignore null
 159         static final boolean ignoreNull = false;
 160
 161         static final boolean switchSpeciesGroup = true;
 162
 163         static boolean useClassification = true;
 164
 165         static boolean isSplitTdwgCodes = false;
 166         static boolean useEmAreaVocabulary = true;
 167
 168         private final boolean removeHttpMapsAnchor = true;
 169
 170
 171         static final String infrGenericRankAbbrev = "[unranked]";
 172         static final String infrSpecificRankAbbrev = "[unranked]";
 173
 174         static boolean useLastScrutinyAsSec = true;
 175         static boolean warnForDifferingSynonymReference = false;
 176
 177
 178         static String taxonTable = "v_cdm_exp_taxaAll";
 179         static String classificationQuery = " SELECT DISTINCT t.PTRefFk, r.RefCache FROM PTaxon t INNER JOIN Reference r ON t.PTRefFk = r.RefId WHERE t.PTRefFk = " + sourceSecId;
 180         static String relPTaxonIdQuery = " SELECT r.RelPTaxonId "
 181                                         + " FROM RelPTaxon AS r "
 182                                         + "   INNER JOIN v_cdm_exp_taxaDirect AS a ON r.PTNameFk2 = a.PTNameFk AND r.PTRefFk2 = a.PTRefFk "
 183                                         + "   INNER JOIN PTaxon As pt1 ON pt1.PTNameFk = r.PTNameFk1 AND pt1.PTRefFk = r.PTRefFk1 "
 184                                         + " WHERE r.RelPTaxonID NOT IN (1874890,1874959,1874932,1874793,1874956,1874971,1874902,1874696) " //Relations to unpublished Kew genus taxa of Bethulaceae which are not imported anymore, but Bethalaceae is still imported
 185                                         + "     AND NOT (pt1.PTRefFk = 8000000 AND pt1.publishFlag = 0) ";
 186         static String nameIdTable = " v_cdm_exp_namesAll ";
 187         static String referenceIdTable = " v_cdm_exp_refAll ";
 188         static String refDetailFilter =  " RefDetailID IN (SELECT RefDetailID FROM v_cdm_exp_RefDetail) ";
 189         static String factFilter = " factId IN ( SELECT factId FROM v_cdm_exp_factsAll WHERE FactCategoryFk NOT IN (12, 13, 14, 249, 251))";
 190         static String occurrenceFilter = " occurrenceId IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";
 191         static String occurrenceSourceFilter = " occurrenceFk IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";
 192         static String commonNameFilter = " commonNameId IN ( SELECT commonNameId FROM v_cdm_exp_commonNamesAll )";
 193         static String webMarkerFilter = " TableNameFk <> 500 OR ( RIdentifierFk IN (SELECT RIdentifier FROM v_cdm_exp_taxaAll)) ";
 194         static String authorTeamFilter =  " authorTeamId IN (SELECT drvTab.authorTeamId FROM (SELECT authorTeamId FROM v_cdm_exp_authorTeamsAll) as drvTab) ";
 195         static String authorFilter = " authorId IN (SELECT drvTab.authorId FROM (SELECT authorId FROM v_cdm_exp_authorsAll) as drvTab) ";
 196
 197
 198
 199 // **************** ALL *********************
 200
 201
 202         public void importEm2CDM (Source source, ICdmDataSource destination, DbSchemaValidation hbm2dll){
 203                 System.out.println("Start import from BerlinModel("+ berlinModelSource.getDatabase() + ") to " + cdmDestination.getDatabase() + " ...");
 204                 //make BerlinModel Source
 205
 206                 BerlinModelImportConfigurator config = BerlinModelImportConfigurator.NewInstance(source,  destination);
 207
 208                 config.setClassificationName(classificationName);
 209
 210                 config.setClassificationUuid(classificationUuid);
 211                 config.setSourceSecId(sourceSecId);
 212                 config.setNomenclaturalCode(nomenclaturalCode);
 213                 config.setIgnoreNull(ignoreNull);
 214
 215                 config.setDoAuthors(doAuthors ^ invers);
 216                 config.setDoReferences(invers ? doReferences.invers() : doReferences);
 217                 config.setDoTaxonNames(doTaxonNames ^ invers);
 218                 config.setDoRelNames(doRelNames ^ invers);
 219                 config.setDoNameStatus(doNameStatus ^ invers);
 220                 config.setDoTypes(doTypes);  //always false
 221                 config.setDoNameFacts(doNameFacts ^ invers);
 222                 config.setDoTaxa(doTaxa ^ invers);
 223                 config.setDoRelTaxa(doRelTaxa ^ invers);
 224                 config.setDoFacts(doFacts ^ invers);
 225                 config.setDoOccurrence(doOccurrences ^ invers);
 226                 config.setDoOccurrenceSources(doOccurrenceSources ^ invers);
 227         config.setDoCommonNames(doCommonNames ^ invers);
 228         config.setDoNamedAreas(doNamedAreas ^ invers);
 229
 230                 config.setDoMarker(doMarker);
 231                 config.setDoUser(doUser ^ invers);
 232
 233                 config.setEuroMed(true);
 234                 config.setDoSourceNumber(true);
 235
 236                 config.setLogNotMatchingOldNames(logNotMatchingOldNames);
 237                 config.setLogMatchingNotExportedOldNames(logMatchingNotExportedOldNames);
 238                 config.setCheckOldNameIsSynonym(checkOldNameIsSynonym);
 239                 config.setIncludeMANsForOldNameCheck(includeMANsForOldNameCheck);
 240
 241                 config.setUseClassification(useClassification);
 242                 config.setSourceRefUuid(BerlinModelTransformer.uuidSourceRefEuroMed);
 243                 config.setEditor(editor);
 244                 config.setDbSchemaValidation(hbm2dll);
 245                 config.setUseLastScrutinyAsSec(useLastScrutinyAsSec);
 246                 config.setWarnForDifferingSynonymReference(warnForDifferingSynonymReference);
 247
 248                 // maximum number of name facts to import
 249                 config.setMaximumNumberOfNameFacts(maximumNumberOfNameFacts);
 250
 251                 config.setInfrGenericRankAbbrev(infrGenericRankAbbrev);
 252                 config.setInfrSpecificRankAbbrev(infrSpecificRankAbbrev);
 253                 config.setRemoveHttpMapsAnchor(removeHttpMapsAnchor);
 254
 255                 config.setDoPreliminaryRefDetailsWithNames(doPreliminaryRefDetailsWithNames);
 256
 257 //              filter
 258                 config.setTaxonTable(taxonTable);
 259                 config.setClassificationQuery(classificationQuery);
 260                 config.setRelTaxaIdQuery(relPTaxonIdQuery);
 261                 config.setNameIdTable(nameIdTable);
 262                 config.setReferenceIdTable(referenceIdTable);
 263                 config.setAuthorTeamFilter(authorTeamFilter);
 264                 config.setAuthorFilter(authorFilter);
 265                 config.setFactFilter(factFilter);
 266                 config.setRefDetailFilter(refDetailFilter);
 267                 config.setCommonNameFilter(commonNameFilter);
 268                 config.setOccurrenceFilter(occurrenceFilter);
 269                 config.setOccurrenceSourceFilter(occurrenceSourceFilter);
 270                 config.setWebMarkerFilter(webMarkerFilter);
 271                 config.setUseSingleClassification(useSingleClassification);
 272
 273                 //TDWG codes
 274                 config.setSplitTdwgCodes(isSplitTdwgCodes);
 275                 config.setUseEmAreaVocabulary(useEmAreaVocabulary);
 276
 277                 config.setCheck(check);
 278                 config.setEditor(editor);
 279                 config.setRecordsPerTransaction(partitionSize);
 280
 281                 config.setSwitchSpeciesGroup(switchSpeciesGroup);
 282
 283                 // invoke import
 284                 CdmDefaultImport<BerlinModelImportConfigurator> bmImport = new CdmDefaultImport<>();
 285                 bmImport.invoke(config);
 286
 287 //              renameRanks(config, bmImport);
 288
 289                 createFeatureTree(config, bmImport);
 290
 291                 changeCommonNameLabel(config, bmImport);
 292
 293                 createUsersAndRoles(config, bmImport);
 294
 295         runTransmissionEngine(config, bmImport);
 296
 297         importShapefile(config, bmImport);
 298
 299 //      createPreferences(config, bmImport); => manual
 300
 301 //        markAreasAsHidden(config, bmImport);  //has been moved to BM occurrence import
 302
 303                 System.out.println("End import from BerlinModel ("+ source.getDatabase() + ")...");
 304         }
 305
 306     private void createPreferences(BerlinModelImportConfigurator config,
 307             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 308
 309         if (config.isDoUser() && (config.getCheck().isImport() )){
 310             ICdmRepository app = bmImport.getCdmAppController();
 311
 312             //area vocs
 313             CdmPreference preference = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AvailableDistributionAreaVocabularies, BerlinModelTransformer.uuidVocCaucasusAreas.toString()+";" + BerlinModelTransformer.uuidVocEuroMedAreas.toString());
 314             preference.setAllowOverride(false);
 315             app.getPreferenceService().set(preference);
 316
 317             //occ status list
 318             String status ="42946bd6-9c22-45ad-a910-7427e8f60bfd;9eb99fe6-59e2-4445-8e6a-478365bd0fa9;c3ee7048-15b7-4be1-b687-9ce9c1a669d6;643cf9d1-a5f1-4622-9837-82ef961e880b;0c54761e-4887-4788-9dfa-7190c88746e3;83eb0aa0-1a45-495a-a3ca-bf6958b74366;aeec2947-2700-4623-8e32-9e3a430569d1;ddeac4f2-d8fa-43b8-ad7e-ca13abdd32c7;310373bf-7df4-4d02-8cb3-bcc7448805fc;5c397f7b-59ef-4c11-a33c-45691ceda91b;925662c1-bb10-459a-8c53-da5a738ac770;61cee840-801e-41d8-bead-015ad866c2f1;e191e89a-a751-4b0c-b883-7f1de70915c9";
 319             CdmPreference statusListPref = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AvailableDistributionAreaVocabularies, status);
 320             statusListPref.setAllowOverride(false);
 321             app.getPreferenceService().set(statusListPref);
 322
 323             //distr. editor activated
 324             CdmPreference distrEditorActive = CdmPreference.NewTaxEditorInstance(PreferencePredicate.DistributionEditorActivated, "true");
 325             statusListPref.setAllowOverride(true);
 326             app.getPreferenceService().set(distrEditorActive);
 327
 328 //            //idInVoc for areas
 329 //            CdmPreference distrEditorShowIdInVocForAreas = CdmPreference.NewTaxEditorInstance(PreferencePredicate.Sho.ShowIdInVocabulary, "true");
 330 //            distrEditorShowIdInVocForAreas.setAllowOverride(true);
 331 //            app.getPreferenceService().set(distrEditorShowIdInVocForAreas);
 332 //
 333 //            //areas sort order
 334 //            //?? correct?
 335 //            CdmPreference distrEditorSorted = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AreasSortedByIdInVocabulary, "true");
 336 //            distrEditorSorted.setAllowOverride(true);
 337 //            app.getPreferenceService().set(distrEditorSorted);
 338 //
 339 //            //distr. status uses symbol
 340 //            //?? correct?
 341 //            CdmPreference distrEditorStatusUseSymbols = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowSymbolForStatus, "false");
 342 //            distrEditorStatusUseSymbols.setAllowOverride(true);
 343 //            app.getPreferenceService().set(distrEditorStatusUseSymbols);
 344
 345             //media view
 346             CdmPreference showMediaView = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowMediaView, "false");
 347             showMediaView.setAllowOverride(false);
 348             app.getPreferenceService().set(showMediaView);
 349
 350             //multi classification
 351             CdmPreference multiClassification = CdmPreference.NewTaxEditorInstance(PreferencePredicate.DisableMultiClassification, "true");
 352             multiClassification.setAllowOverride(false);
 353             app.getPreferenceService().set(multiClassification);
 354
 355             //taxon node wizard
 356             CdmPreference showTaxonNodeWizard = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowTaxonNodeWizard, "false");
 357             showTaxonNodeWizard.setAllowOverride(false);
 358             app.getPreferenceService().set(showTaxonNodeWizard);
 359
 360             //import+export
 361             CdmPreference showImportExportMenu = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowImportExportMenu, "false");
 362             showImportExportMenu.setAllowOverride(true);
 363             app.getPreferenceService().set(showImportExportMenu);
 364
 365             //show specimen
 366             CdmPreference showSpecimen = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowSpecimen, "false");
 367             showSpecimen.setAllowOverride(false);
 368             app.getPreferenceService().set(showSpecimen);
 369
 370         }
 371
 372     }
 373
 374     //Rename Ranks (still needed?)
 375     private void renameRanks(BerlinModelImportConfigurator config,
 376             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 377
 378         if (config.isDoTaxonNames() && (config.getCheck().isImport() )  ){
 379                         ICdmRepository app = bmImport.getCdmAppController();
 380                         TransactionStatus tx = app.startTransaction();
 381                         try {
 382                                 Rank sectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());
 383                                 Representation repr = sectBot.getRepresentation(Language.ENGLISH());
 384                                 repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());
 385                                 repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());
 386                                 sectBot.setTitleCache(null, false);  //to definitely update the titleCache also
 387                                 app.getTermService().saveOrUpdate(sectBot);
 388
 389                                 Rank subSectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());
 390                                 repr = subSectBot.getRepresentation(Language.ENGLISH());
 391                                 repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());
 392                                 repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());
 393                                 subSectBot.setTitleCache(null, false);  //to definitely update the titleCache also
 394                                 app.getTermService().saveOrUpdate(subSectBot);
 395                                 app.commitTransaction(tx);
 396                         } catch (Exception e) {
 397                             e.printStackTrace();
 398                 logger.error("Exception in renameRanks: " + e.getMessage());
 399                         }
 400                 }
 401     }
 402
 403     //create feature tree
 404     private void createFeatureTree(BerlinModelImportConfigurator config,
 405             CdmDefaultImport<BerlinModelImportConfigurator> bmImport){
 406             if (config.isDoFacts() && (config.getCheck().isImport()  )  ){
 407                         try {
 408                 ICdmRepository app = bmImport.getCdmAppController();
 409                 TransactionStatus tx = app.startTransaction();
 410
 411                 //make feature tree
 412                 TermTree<Feature> tree = TreeCreator.flatTree(featureTreeUuid, config.getFeatureMap(), featureKeyList);
 413                 tree.setTitleCache("Euro+Med Feature Tree", true);
 414                 tree.getRoot().addChild(Feature.IMAGE());
 415                 tree.getRoot().addChild(Feature.DISTRIBUTION(), 1);
 416                 tree.getRoot().addChild(Feature.COMMON_NAME(), 2);
 417                 app.getFeatureTreeService().saveOrUpdate(tree);
 418
 419                 app.commitTransaction(tx);
 420             } catch (Exception e) {
 421                 e.printStackTrace();
 422                 logger.error("Exception in createFeatureTree: " + e.getMessage());
 423             }
 424                 }
 425     }
 426
 427     //Change common name label
 428     private void changeCommonNameLabel(BerlinModelImportConfigurator config,
 429             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 430             if (config.isDoFacts() && (config.getCheck().isImport()  )  ){
 431                 try {
 432                 ICdmRepository app = bmImport.getCdmAppController();
 433                 TransactionStatus tx = app.startTransaction();
 434
 435                 DefinedTermBase<?> commonNameFeature = app.getTermService().find(Feature.COMMON_NAME().getUuid());
 436                 commonNameFeature.setLabel("Common Names", Language.ENGLISH());
 437                 commonNameFeature.setTitleCache(null, false);  //to definitely update the titleCache also
 438                 app.getTermService().saveOrUpdate(commonNameFeature);
 439
 440                 app.commitTransaction(tx);
 441             } catch (Exception e) {
 442                 e.printStackTrace();
 443                 logger.error("Exception in changeCommonNameLabel: " + e.getMessage());
 444             }
 445             }
 446     }
 447
 448     //1. run transmission engine #3979
 449     private void runTransmissionEngine(BerlinModelImportConfigurator config,
 450             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 451
 452         if (doRunTransmissionEngine && (config.getCheck().isImport()  )  ){
 453             try {
 454                 ICdmRepository app = bmImport.getCdmAppController();
 455
 456                 final List<String> term_init_strategy = Arrays.asList(new String []{
 457                         "representations"
 458                 });
 459
 460                 UUID uuidSuperAreaLevel = BerlinModelTransformer.uuidEuroMedAreaLevelFirst;
 461                 NamedAreaLevel euroMedLevel1 = (NamedAreaLevel)app.getTermService().find(uuidSuperAreaLevel);
 462
 463                 Pager<NamedArea> areaPager = app.getTermService().list(
 464                         euroMedLevel1,
 465                         (NamedAreaType) null,
 466                         null,
 467                         null,
 468                         (List<OrderHint>) null,
 469                         term_init_strategy);
 470                 DistributionAggregation distributionAggregation = new DistributionAggregation();
 471
 472                 Set<UUID> classificationUuids = new HashSet<>();
 473                 TaxonNodeFilter filter = TaxonNodeFilter.NewInstance(classificationUuids, null, null, null, null,
 474                         null, Rank.uuidGenus);
 475
 476                 DistributionAggregationConfiguration aggregationConfig = DistributionAggregationConfiguration
 477                         .NewInstance(
 478                         AggregationMode.byWithinTaxonAndToParent(),
 479                         areaPager.getRecords().stream().map(area->area.getUuid()).collect(Collectors.toList()),
 480                         filter,
 481                         DefaultProgressMonitor.NewInstance());
 482                 distributionAggregation.invoke(aggregationConfig, app);
 483             } catch (Exception e) {
 484                 e.printStackTrace();
 485                 logger.error("Exception in markAreasAsHidden: " + e.getMessage());
 486             }
 487         }
 488     }
 489
 490 //    //5.Mark areas to be hidden #3979 .5
 491 //    private void markAreasAsHidden(BerlinModelImportConfigurator config,
 492 //            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 493 //
 494 //        if (config.isDoOccurrence() && (config.getCheck().isImport())){
 495 //              try {
 496 //                ICdmRepository app = bmImport.getCdmAppController();
 497 //                TransactionStatus tx = app.startTransaction();
 498 //
 499 //                MarkerType hiddenAreaMarkerType = MarkerType.NewInstance("Used to hide distributions for the named areas in publications", "Hidden Area", null);
 500 //                hiddenAreaMarkerType.setUuid(BerlinModelTransformer.uuidHiddenArea);
 501 //                @SuppressWarnings("unchecked")
 502 //                TermVocabulary<MarkerType> vocUserDefinedMarkerTypes = app.getVocabularyService().find(CdmImportBase.uuidUserDefinedMarkerTypeVocabulary);
 503 //                if (vocUserDefinedMarkerTypes == null){
 504 //                    String message = "Marker type vocabulary could not be found. Hidden areas not added.";
 505 //                    logger.error(message);
 506 //                    System.out.println(message);
 507 //                }else{
 508 //                    vocUserDefinedMarkerTypes.addTerm(hiddenAreaMarkerType);
 509 //                    app.getVocabularyService().saveOrUpdate(vocUserDefinedMarkerTypes);
 510 //
 511 //                    //Add hidden area marker to Rs(C) and Rs(N)
 512 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs);
 513 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_B);
 514 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_C);
 515 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_E);
 516 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_N);
 517 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_K);
 518 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_W);
 519 //                 }
 520 //                app.commitTransaction(tx);
 521 //            } catch (Exception e) {
 522 //                e.printStackTrace();
 523 //                logger.error("Exception in markAreasAsHidden: " + e.getMessage());
 524 //            }
 525 //          }
 526 //    }
 527 //
 528 //    private void hideArea(ICdmRepository app, MarkerType hiddenAreaMarkerType, UUID areaUuid) {
 529 //        NamedArea area = (NamedArea)app.getTermService().find(areaUuid);
 530 //        area.addMarker(Marker.NewInstance(hiddenAreaMarkerType, true));
 531 //        app.getTermService().saveOrUpdate(area);
 532 //    }
 533
 534     //2. import shapefile attributes #3979 .2
 535     private void importShapefile(BerlinModelImportConfigurator config,
 536             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 537
 538         if (config.isDoOccurrence() && (config.getCheck().isImport())){
 539
 540                try {
 541                    UUID areaVocabularyUuid = BerlinModelTransformer.uuidVocEuroMedAreas;
 542                List<String> idSearchFields = Arrays.asList(new String[]{"EMAREA","PARENT"});
 543                String wmsLayerName = "euromed_2013";
 544                Set<UUID> areaUuidSet = null;
 545
 546                ICdmRepository app = bmImport.getCdmAppController();
 547                IEditGeoService geoService = (IEditGeoService)app.getBean("editGeoService");
 548
 549                Map<NamedArea, String> resultMap;
 550                try {
 551                    InputStream in = EuroMedActivator.class.getResourceAsStream("/euromed/euromed_2013.csv");
 552                    Reader reader = new InputStreamReader(in, "UTF-8");
 553
 554                    resultMap = geoService.mapShapeFileToNamedAreas(
 555                                reader, idSearchFields , wmsLayerName , areaVocabularyUuid, areaUuidSet);
 556                    Map<String, String> flatResultMap = new HashMap<>(resultMap.size());
 557                    for(NamedArea area : resultMap.keySet()){
 558                        flatResultMap.put(area.getTitleCache() + " [" + area.getUuid() + "]", resultMap.get(area));
 559                    }
 560                } catch (IOException e) {
 561                     String message = "IOException when reading from mapping file or creating result map.";
 562                     logger.error(message);
 563                     System.out.println(message);
 564                }
 565             } catch (Exception e) {
 566                 e.printStackTrace();
 567                 logger.error("Exception in importShapefile: " + e.getMessage());
 568             }
 569             }
 570     }
 571
 572     //4. Create users and assign roles  #3979
 573     private void createUsersAndRoles(BerlinModelImportConfigurator config,
 574             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 575
 576         try {
 577             if (config.isDoRelTaxa() && (config.getCheck().isImport())){
 578                 ICdmRepository app = bmImport.getCdmAppController();
 579                 TransactionStatus tx = app.startTransaction();
 580
 581                 //eraabstraube
 582                 String eraabstraube = "e.raabstraube";
 583                 List<User> users = app.getUserService().listByUsername(eraabstraube, MatchMode.EXACT, null, null, null, null, null);
 584                 User userEraabStraube;
 585                 if (users.isEmpty()){
 586                     userEraabStraube = User.NewInstance(eraabstraube, eraabstraube);
 587                 }else{
 588                     userEraabStraube = users.get(0);
 589                 }
 590                 if (userEraabStraube.getPerson() == null){
 591                     Person eckhard = Person.NewInstance();
 592                     eckhard.setFamilyName("von Raab-Straube");
 593                     eckhard.setGivenName("Eckhard");
 594                     eckhard.setPrefix("Dr.");
 595                     userEraabStraube.setPerson(eckhard);
 596                 }
 597                 app.getUserService().saveOrUpdate(userEraabStraube);
 598
 599                 //groups
 600                 Group groupEditor = app.getGroupService().load(Group.GROUP_EDITOR_UUID);
 601                 groupEditor.addMember(userEraabStraube);
 602                 app.getGroupService().saveOrUpdate(groupEditor);
 603
 604                 Group groupProjectManager = app.getGroupService().load(Group.GROUP_PROJECT_MANAGER_UUID);
 605                 groupProjectManager.addMember(userEraabStraube);
 606                 app.getGroupService().saveOrUpdate(groupProjectManager);
 607
 608                 String[] publishRoles = new String[]{Role.ROLE_PUBLISH.toString()};
 609                 Group groupPublisher = checkGroup(app.getGroupService(), Group.GROUP_PUBLISH_UUID, "Publisher", publishRoles);
 610                 groupPublisher.addMember(userEraabStraube);
 611                 app.getGroupService().saveOrUpdate(groupPublisher);
 612
 613                 UUID uuidEuroMedPlantBaseGroup = UUID.fromString("91be42ea-ad04-4458-9836-389277e773db");
 614                 String[] emPlantBaseRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]"};
 615                 Group euroMedPlantbase = checkGroup(app.getGroupService(), uuidEuroMedPlantBaseGroup, "Euro+Med Plantbase", emPlantBaseRoles);
 616                 euroMedPlantbase.addMember(userEraabStraube);
 617                 app.getGroupService().saveOrUpdate(euroMedPlantbase);
 618
 619                 //cichorieae-editor
 620                 String cichorieaeEditor = "cichorieae-editor";
 621                 app.getUserService().listByUsername(cichorieaeEditor, MatchMode.EXACT, null, null, null, null, null);
 622                 User userCichEditor;
 623                 if (users.isEmpty()){
 624                     userCichEditor = User.NewInstance(cichorieaeEditor, cichorieaeEditor);
 625                 }else{
 626                     userCichEditor = users.get(0);
 627                 }
 628                 app.getUserService().saveOrUpdate(userCichEditor);
 629
 630                 //groups
 631                 groupEditor.addMember(userCichEditor);
 632                 app.getGroupService().saveOrUpdate(groupEditor);
 633
 634                 UUID uuidCichorieaeSubtree = null;
 635                 UUID uuidCichorieae = UUID.fromString("63c7dbeb-b9a2-48b8-a75f-e3fe5e161f7c");
 636                 Taxon cich = (Taxon)app.getTaxonService().find(uuidCichorieae);
 637                 if (cich != null){
 638                     if (!cich.getTaxonNodes().isEmpty()){
 639                         TaxonNode cichNode = cich.getTaxonNodes().iterator().next();
 640                         uuidCichorieaeSubtree = cichNode.getUuid();
 641                     }
 642                 }
 643
 644                 String[] cichorieaeRoles = new String[]{};
 645                 if (uuidCichorieaeSubtree != null){
 646                     cichorieaeRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]{"+uuidCichorieaeSubtree.toString()+"}"};
 647                 }else{
 648                     String message = "Cichorieae node could not be found for cichorieae-editor role";
 649                     logger.warn(message);
 650                     System.out.println(message);
 651                 }
 652                 UUID uuidCichorieaeGroup = UUID.fromString("a630938d-dd4f-48c2-9406-91def487b11e");
 653                 String cichorieaeGroupName = "Cichorieae";
 654                 Group cichorieaeGroup = checkGroup(app.getGroupService(), uuidCichorieaeGroup, cichorieaeGroupName, cichorieaeRoles);
 655                 cichorieaeGroup.addMember(userCichEditor);
 656                 app.getGroupService().saveOrUpdate(cichorieaeGroup);
 657
 658                 app.commitTransaction(tx);
 659             }
 660         } catch (Exception e) {
 661             e.printStackTrace();
 662             logger.error("Exception in createUsersAndRoles: " + e.getMessage());
 663         }
 664     }
 665
 666         /**
 667           * copied from {@link FirstDataInserter#checkGroup}
 668      */
 669     private Group checkGroup(IGroupService groupService, UUID groupUuid, String groupName, String[] requiredAuthorities) {
 670         Group group = groupService.load(groupUuid);
 671         if(group == null){
 672             group = Group.NewInstance();
 673             group.setUuid(groupUuid);
 674             logger.info("New Group '" + groupName + "' created");
 675         }
 676         group.setName(groupName); // force name
 677
 678         Set<GrantedAuthority> grantedAuthorities = group.getGrantedAuthorities();
 679
 680         for(String a : requiredAuthorities){
 681             boolean isMissing = true;
 682             for(GrantedAuthority ga : grantedAuthorities){
 683                 if(a.equals(ga.getAuthority())){
 684                     isMissing = false;
 685                     break;
 686                 }
 687             }
 688             if(isMissing){
 689                 GrantedAuthorityImpl newGa = GrantedAuthorityImpl.NewInstance(a);
 690                 group.addGrantedAuthority(newGa);
 691                 logger.info("New GrantedAuthority '" + a + "' added  to '" + groupName + "'");
 692             }
 693         }
 694         groupService.saveOrUpdate(group);
 695         logger.info("Check of group  '" + groupName + "' done");
 696         return group;
 697     }
 698
 699
 700         /**
 701          * @param args
 702          */
 703         public static void main(String[] args) {
 704                 EuroMedActivator importActivator = new EuroMedActivator();
 705                 Source source = berlinModelSource;
 706                 ICdmDataSource cdmRepository = CdmDestinations.chooseDestination(args) != null ? CdmDestinations.chooseDestination(args) : cdmDestination;
 707
 708                 importActivator.importEm2CDM(source, cdmRepository, hbm2dll);
 709                 System.exit(0);
 710         }
 711
 712 }