app-import/src/main/java/eu/etaxonomy/cdm/app/berlinModelImport/CaucasusEuroMedActivator.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.app.berlinModelImport;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.io.InputStreamReader;
  15 import java.io.Reader;
  16 import java.util.Arrays;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import org.apache.log4j.Logger;
  24 import org.springframework.security.core.GrantedAuthority;
  25 import org.springframework.transaction.TransactionStatus;
  26
  27 import eu.etaxonomy.cdm.api.application.FirstDataInserter;
  28 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  29 import eu.etaxonomy.cdm.api.service.IGroupService;
  30 import eu.etaxonomy.cdm.api.service.description.TransmissionEngineDistribution;
  31 import eu.etaxonomy.cdm.api.service.description.TransmissionEngineDistribution.AggregationMode;
  32 import eu.etaxonomy.cdm.api.service.pager.Pager;
  33 import eu.etaxonomy.cdm.app.common.CdmDestinations;
  34 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  35 import eu.etaxonomy.cdm.database.DbSchemaValidation;
  36 import eu.etaxonomy.cdm.database.ICdmDataSource;
  37 import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
  38 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
  39 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
  40 import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
  41 import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
  42 import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;
  43 import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
  44 import eu.etaxonomy.cdm.io.common.Source;
  45 import eu.etaxonomy.cdm.model.agent.Person;
  46 import eu.etaxonomy.cdm.model.common.GrantedAuthorityImpl;
  47 import eu.etaxonomy.cdm.model.common.Group;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.common.User;
  50 import eu.etaxonomy.cdm.model.description.Feature;
  51 import eu.etaxonomy.cdm.model.location.NamedArea;
  52 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  53 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  54 import eu.etaxonomy.cdm.model.metadata.CdmPreference;
  55 import eu.etaxonomy.cdm.model.metadata.PreferencePredicate;
  56 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  57 import eu.etaxonomy.cdm.model.name.Rank;
  58 import eu.etaxonomy.cdm.model.taxon.Taxon;
  59 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  60 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  61 import eu.etaxonomy.cdm.model.term.Representation;
  62 import eu.etaxonomy.cdm.model.term.TermTree;
  63 import eu.etaxonomy.cdm.persistence.hibernate.permission.Role;
  64 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  65 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  66
  67
  68 /**
  69  * TODO add the following to a wiki page:
  70  * HINT: If you are about to import into a mysql data base running under windows and if you wish to
  71  * dump and restore the resulting data bas under another operation systen
  72  * you must set the mysql system variable lower_case_table_names = 0 in order to create data base with table compatible names.
  73  *
  74  * @author a.mueller
  75  */
  76 public class CaucasusEuroMedActivator {
  77         private static final Logger logger = Logger.getLogger(CaucasusEuroMedActivator.class);
  78
  79         //database validation status (create, update, validate ...)
  80         static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
  81         static final Source berlinModelSource = BerlinModelSources.euroMed_BGBM42();
  82
  83 //    static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
  84     static final ICdmDataSource cdmDestination = CdmDestinations.cdm_production_euromed_caucasus();
  85 //    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed_caucasus();
  86 //      static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_euroMed();
  87
  88     //check - import
  89     static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
  90
  91     static final boolean doUser = true;
  92 //  //authors
  93     static final boolean doAuthors = true;
  94     //references
  95     static final DO_REFERENCES doReferences =  DO_REFERENCES.ALL;
  96     //names
  97     static final boolean doTaxonNames = true;
  98     static final boolean doRelNames = true;
  99     static final boolean doNameStatus = true;
 100     static final boolean doNameFacts = true;
 101
 102     //taxa
 103     static final boolean doTaxa = true;
 104     static final boolean doRelTaxa = true;
 105
 106     static final boolean doNamedAreas = true;
 107
 108     //not relevant for caucasus
 109     static final boolean doFacts = false;
 110     static final boolean doOccurrences = false;
 111     static final boolean doOccurrenceSources =false;
 112     static final boolean doCommonNames = false;  //currently takes very long
 113
 114   //serious types do not exist in E+M except for name types which are handled in name relations
 115     static final boolean doTypes = false;  //serious types do not exist in E+M except for name types which are handled in name relations
 116
 117
 118     static final boolean doRunTransmissionEngine = false; // (hbm2dll == DbSchemaValidation.VALIDATE);
 119
 120     //etc.
 121     static final boolean doMarker = false;  //no relevant markers exist
 122
 123     boolean invers = !(hbm2dll == DbSchemaValidation.CREATE);
 124
 125     boolean doPreliminaryRefDetailsWithNames = true;
 126
 127     boolean logNotMatchingOldNames = false;
 128     boolean logMatchingNotExportedOldNames = false;  //true
 129     boolean checkOldNameIsSynonym = true;
 130
 131         static final boolean includePesiExport = false;
 132
 133         static final int sourceSecId = 7000000; //500000
 134         static final UUID classificationUuid = UUID.fromString("314a68f9-8449-495a-91c2-92fde8bcf344");
 135         static final boolean useSingleClassification = true;
 136         static final String classificationName = "Euro+Med (Caucasus) 2018";
 137         static final UUID featureTreeUuid = UUID.fromString("6a5e1c2b-ec0d-46c8-9c7d-a2059267ffb7");
 138         static final Object[] featureKeyList = new Integer[]{1, 31, 4, 98, 41};
 139
 140         // set to zero for unlimited nameFacts
 141         static final int maximumNumberOfNameFacts = 0;
 142
 143         static final int partitionSize = 2500;
 144
 145
 146         //editor - import
 147         static final EDITOR editor = EDITOR.EDITOR_AS_EDITOR;
 148
 149         //NomenclaturalCode
 150         static final NomenclaturalCode nomenclaturalCode = NomenclaturalCode.ICNAFP;
 151
 152         //ignore null
 153         static final boolean ignoreNull = false;
 154
 155         static final boolean switchSpeciesGroup = true;
 156
 157         static boolean useClassification = true;
 158
 159         static boolean isSplitTdwgCodes = false;
 160         static boolean useEmAreaVocabulary = true;
 161
 162         private final boolean removeHttpMapsAnchor = true;
 163
 164
 165         static final String infrGenericRankAbbrev = "[unranked]";
 166         static final String infrSpecificRankAbbrev = "[unranked]";
 167
 168         static boolean useLastScrutinyAsSec = true;
 169         static boolean warnForDifferingSynonymReference = false;
 170
 171
 172         static String taxonTable = "v_cdm_exp_taxaAll";
 173         static String classificationQuery = " SELECT DISTINCT t.PTRefFk, r.RefCache FROM PTaxon t INNER JOIN Reference r ON t.PTRefFk = r.RefId WHERE t.PTRefFk = " + sourceSecId;
 174         static String relPTaxonIdQuery = " SELECT r.RelPTaxonId "
 175                                         + " FROM RelPTaxon AS r "
 176                                         + "   INNER JOIN v_cdm_exp_taxaDirect AS a ON r.PTNameFk2 = a.PTNameFk AND r.PTRefFk2 = a.PTRefFk "
 177                                         + "   INNER JOIN PTaxon As pt1 ON pt1.PTNameFk = r.PTNameFk1 AND pt1.PTRefFk = r.PTRefFk1 "
 178                                         + " WHERE r.RelPTaxonID NOT IN (1874890,1874959,1874932,1874793,1874956,1874971,1874902,1874696) " //Relations to unpublished Kew genus taxa of Bethulaceae which are not imported anymore, but Bethulaceae is still imported
 179                                         + "     AND NOT (pt1.PTRefFk = 8000000 AND pt1.publishFlag = 0) ";
 180         static String nameIdTable = " v_cdm_exp_namesAll ";
 181         static String referenceIdTable = " v_cdm_exp_refAll ";
 182         static String refDetailFilter =  " RefDetailID IN (SELECT RefDetailID FROM v_cdm_exp_RefDetail) ";
 183         static String factFilter = " factId IN ( SELECT factId FROM v_cdm_exp_factsAll WHERE FactCategoryFk NOT IN (12, 14, 249, 251))";
 184         static String occurrenceFilter = " occurrenceId IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";
 185         static String occurrenceSourceFilter = " occurrenceFk IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";
 186         static String commonNameFilter = " commonNameId IN ( SELECT commonNameId FROM v_cdm_exp_commonNamesAll )";
 187         static String webMarkerFilter = " TableNameFk <> 500 OR ( RIdentifierFk IN (SELECT RIdentifier FROM v_cdm_exp_taxaAll)) ";
 188         static String authorTeamFilter =  " authorTeamId IN (SELECT drvTab.authorTeamId FROM (SELECT authorTeamId FROM v_cdm_exp_authorTeamsAll) as drvTab) ";
 189         static String authorFilter = " authorId IN (SELECT drvTab.authorId FROM (SELECT authorId FROM v_cdm_exp_authorsAll) as drvTab) ";
 190
 191
 192
 193 // **************** ALL *********************
 194
 195
 196
 197
 198         public void importEm2CDM (Source source, ICdmDataSource destination, DbSchemaValidation hbm2dll){
 199                 System.out.println("Start import from BerlinModel("+ berlinModelSource.getDatabase() + ") to " + cdmDestination.getDatabase() + " ...");
 200                 //make BerlinModel Source
 201
 202                 BerlinModelImportConfigurator config = BerlinModelImportConfigurator.NewInstance(source,  destination);
 203
 204                 config.setClassificationName(classificationName);
 205
 206                 config.setClassificationUuid(classificationUuid);
 207                 config.setSourceSecId(sourceSecId);
 208                 config.setNomenclaturalCode(nomenclaturalCode);
 209                 config.setIgnoreNull(ignoreNull);
 210
 211                 config.setDoAuthors(doAuthors ^ invers);
 212                 config.setDoReferences(invers ? doReferences.invers() : doReferences);
 213                 config.setDoTaxonNames(doTaxonNames ^ invers);
 214                 config.setDoRelNames(doRelNames ^ invers);
 215                 config.setDoNameStatus(doNameStatus ^ invers);
 216                 config.setDoTypes(doTypes);  //always false
 217                 config.setDoNameFacts(doNameFacts ^ invers);
 218                 config.setDoTaxa(doTaxa ^ invers);
 219                 config.setDoRelTaxa(doRelTaxa ^ invers);
 220                 config.setDoNamedAreas(doNamedAreas ^ invers);
 221
 222                 config.setDoFacts(doFacts);
 223                 config.setDoOccurrence(doOccurrences);
 224                 config.setDoOccurrenceSources(doOccurrenceSources);
 225         config.setDoCommonNames(doCommonNames);
 226
 227                 config.setDoMarker(doMarker);
 228                 config.setDoUser(doUser ^ invers);
 229
 230                 config.setEuroMed(true);
 231                 config.setDoSourceNumber(true);
 232
 233                 config.setLogNotMatchingOldNames(logNotMatchingOldNames);
 234                 config.setLogMatchingNotExportedOldNames(logMatchingNotExportedOldNames);
 235                 config.setCheckOldNameIsSynonym(checkOldNameIsSynonym);
 236
 237                 config.setUseClassification(useClassification);
 238                 config.setSourceRefUuid(BerlinModelTransformer.uuidSourceRefEuroMed);
 239                 config.setEditor(editor);
 240                 config.setDbSchemaValidation(hbm2dll);
 241                 config.setUseLastScrutinyAsSec(useLastScrutinyAsSec);
 242                 config.setWarnForDifferingSynonymReference(warnForDifferingSynonymReference);
 243
 244                 // maximum number of name facts to import
 245                 config.setMaximumNumberOfNameFacts(maximumNumberOfNameFacts);
 246
 247                 config.setInfrGenericRankAbbrev(infrGenericRankAbbrev);
 248                 config.setInfrSpecificRankAbbrev(infrSpecificRankAbbrev);
 249                 config.setRemoveHttpMapsAnchor(removeHttpMapsAnchor);
 250
 251                 config.setDoPreliminaryRefDetailsWithNames(doPreliminaryRefDetailsWithNames);
 252
 253 //              filter
 254                 config.setTaxonTable(taxonTable);
 255                 config.setClassificationQuery(classificationQuery);
 256                 config.setRelTaxaIdQuery(relPTaxonIdQuery);
 257                 config.setNameIdTable(nameIdTable);
 258                 config.setReferenceIdTable(referenceIdTable);
 259                 config.setAuthorTeamFilter(authorTeamFilter);
 260                 config.setAuthorFilter(authorFilter);
 261                 config.setFactFilter(factFilter);
 262                 config.setRefDetailFilter(refDetailFilter);
 263                 config.setCommonNameFilter(commonNameFilter);
 264                 config.setOccurrenceFilter(occurrenceFilter);
 265                 config.setOccurrenceSourceFilter(occurrenceSourceFilter);
 266                 config.setWebMarkerFilter(webMarkerFilter);
 267                 config.setUseSingleClassification(useSingleClassification);
 268
 269                 //TDWG codes
 270                 config.setSplitTdwgCodes(isSplitTdwgCodes);
 271                 config.setUseEmAreaVocabulary(useEmAreaVocabulary);
 272
 273                 config.setCheck(check);
 274                 config.setEditor(editor);
 275                 config.setRecordsPerTransaction(partitionSize);
 276
 277                 config.setSwitchSpeciesGroup(switchSpeciesGroup);
 278
 279                 // invoke import
 280                 CdmDefaultImport<BerlinModelImportConfigurator> bmImport = new CdmDefaultImport<>();
 281                 bmImport.invoke(config);
 282
 283                 renameRanks(config, bmImport);
 284
 285                 createFeatureTree(config, bmImport);
 286
 287                 changeCommonNameLabel(config, bmImport);
 288
 289                 createUsersAndRoles(config, bmImport);
 290
 291                 createPreferences(config, bmImport);
 292
 293                 System.out.println("End import from BerlinModel ("+ source.getDatabase() + ")...");
 294         }
 295
 296         /**
 297      * @param config
 298      * @param bmImport
 299      */
 300     private void createPreferences(BerlinModelImportConfigurator config,
 301             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 302
 303         if (config.isDoUser() && (config.getCheck().isImport() )){
 304             ICdmRepository app = bmImport.getCdmAppController();
 305             //area voc
 306             CdmPreference areaVocPref = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AvailableDistributionAreaVocabularies, BerlinModelTransformer.uuidVocCaucasusAreas.toString());
 307             areaVocPref.setAllowOverride(false);
 308             app.getPreferenceService().set(areaVocPref);
 309
 310             //occ status list
 311             String status ="42946bd6-9c22-45ad-a910-7427e8f60bfd;9eb99fe6-59e2-4445-8e6a-478365bd0fa9;c3ee7048-15b7-4be1-b687-9ce9c1a669d6;643cf9d1-a5f1-4622-9837-82ef961e880b;0c54761e-4887-4788-9dfa-7190c88746e3;83eb0aa0-1a45-495a-a3ca-bf6958b74366;aeec2947-2700-4623-8e32-9e3a430569d1;ddeac4f2-d8fa-43b8-ad7e-ca13abdd32c7;310373bf-7df4-4d02-8cb3-bcc7448805fc;5c397f7b-59ef-4c11-a33c-45691ceda91b;925662c1-bb10-459a-8c53-da5a738ac770;61cee840-801e-41d8-bead-015ad866c2f1;e191e89a-a751-4b0c-b883-7f1de70915c9";
 312             CdmPreference statusListPref = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AvailableDistributionAreaVocabularies, status);
 313             statusListPref.setAllowOverride(false);
 314             app.getPreferenceService().set(statusListPref);
 315
 316
 317             //distr. editor activated
 318             CdmPreference distrEditorActive = CdmPreference.NewTaxEditorInstance(PreferencePredicate.DistributionEditorActivated, "true");
 319             statusListPref.setAllowOverride(true);
 320             app.getPreferenceService().set(distrEditorActive);
 321
 322 //            //idInVoc for areas
 323 //            CdmPreference distrEditorShowIdInVocForAreas = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowIdInVocabulary, "true");
 324 //            distrEditorShowIdInVocForAreas.setAllowOverride(true);
 325 //            app.getPreferenceService().set(distrEditorShowIdInVocForAreas);
 326 //
 327 //            //areas sort order
 328 //            CdmPreference distrEditorSorted = CdmPreference.NewTaxEditorInstance(PreferencePredicate.AreasSortedByIdInVocabulary, "true");
 329 //            distrEditorSorted.setAllowOverride(true);
 330 //            app.getPreferenceService().set(distrEditorSorted);
 331 //
 332 //            //distr. status uses symbol
 333 //            //?? correct?
 334 //            CdmPreference distrEditorStatusUseSymbols = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowSymbolForStatus, "false");
 335 //            distrEditorStatusUseSymbols.setAllowOverride(true);
 336 //            app.getPreferenceService().set(distrEditorStatusUseSymbols);
 337
 338
 339             //media view
 340             CdmPreference showMediaView = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowMediaView, "false");
 341             showMediaView.setAllowOverride(false);
 342             app.getPreferenceService().set(showMediaView);
 343             //multi classification
 344             CdmPreference multiClassification = CdmPreference.NewTaxEditorInstance(PreferencePredicate.DisableMultiClassification, "false");
 345             multiClassification.setAllowOverride(false);
 346             app.getPreferenceService().set(multiClassification);
 347             //taxon node wizard
 348             CdmPreference showTaxonNodeWizard = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowTaxonNodeWizard, "false");
 349             showTaxonNodeWizard.setAllowOverride(false);
 350             app.getPreferenceService().set(showTaxonNodeWizard);
 351
 352             //import+export
 353             CdmPreference showImportExportMenu = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowImportExportMenu, "false");
 354             showImportExportMenu.setAllowOverride(true);
 355             app.getPreferenceService().set(showImportExportMenu);
 356
 357             //show specimen
 358             CdmPreference showSpecimen = CdmPreference.NewTaxEditorInstance(PreferencePredicate.ShowSpecimen, "false");
 359             showSpecimen.setAllowOverride(false);
 360             app.getPreferenceService().set(showSpecimen);
 361         }
 362     }
 363
 364     //Rename Ranks (still needed?)
 365     private void renameRanks(BerlinModelImportConfigurator config,
 366             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 367
 368         if (config.isDoTaxonNames() && (config.getCheck().isImport() )  ){
 369                         ICdmRepository app = bmImport.getCdmAppController();
 370                         TransactionStatus tx = app.startTransaction();
 371                         try {
 372                                 Rank sectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());
 373                                 Representation repr = sectBot.getRepresentation(Language.ENGLISH());
 374                                 repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());
 375                                 repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());
 376                                 sectBot.setTitleCache(null, false);  //to definitely update the titleCache also
 377                                 app.getTermService().saveOrUpdate(sectBot);
 378
 379                                 Rank subSectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());
 380                                 repr = subSectBot.getRepresentation(Language.ENGLISH());
 381                                 repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());
 382                                 repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());
 383                                 subSectBot.setTitleCache(null, false);  //to definitely update the titleCache also
 384                                 app.getTermService().saveOrUpdate(subSectBot);
 385                                 app.commitTransaction(tx);
 386                         } catch (Exception e) {
 387                             e.printStackTrace();
 388                 logger.error("Exception in renameRanks: " + e.getMessage());
 389                         }
 390                 }
 391     }
 392
 393     //create feature tree
 394     private void createFeatureTree(BerlinModelImportConfigurator config,
 395             CdmDefaultImport<BerlinModelImportConfigurator> bmImport){
 396             if (config.isDoFacts() && (config.getCheck().isImport()  )  ){
 397                         try {
 398                 ICdmRepository app = bmImport.getCdmAppController();
 399                 TransactionStatus tx = app.startTransaction();
 400
 401                 //make feature tree
 402                 TermTree<Feature> tree = TreeCreator.flatTree(featureTreeUuid, config.getFeatureMap(), featureKeyList);
 403                 tree.setTitleCache("Euro+Med Feature Tree", true);
 404                 tree.getRoot().addChild(Feature.IMAGE());
 405                 tree.getRoot().addChild(Feature.DISTRIBUTION(), 1);
 406                 tree.getRoot().addChild(Feature.COMMON_NAME(), 2);
 407                 app.getFeatureTreeService().saveOrUpdate(tree);
 408
 409                 app.commitTransaction(tx);
 410             } catch (Exception e) {
 411                 e.printStackTrace();
 412                 logger.error("Exception in createFeatureTree: " + e.getMessage());
 413             }
 414                 }
 415     }
 416
 417     //Change common name label
 418     private void changeCommonNameLabel(BerlinModelImportConfigurator config,
 419             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 420             if (config.isDoFacts() && (config.getCheck().isImport()  )  ){
 421                 try {
 422                 ICdmRepository app = bmImport.getCdmAppController();
 423                 TransactionStatus tx = app.startTransaction();
 424
 425                 DefinedTermBase<?> commonNameFeature = app.getTermService().find(Feature.COMMON_NAME().getUuid());
 426                 commonNameFeature.setLabel("Common Names", Language.ENGLISH());
 427                 commonNameFeature.setTitleCache(null, false);  //to definitely update the titleCache also
 428                 app.getTermService().saveOrUpdate(commonNameFeature);
 429
 430                 app.commitTransaction(tx);
 431             } catch (Exception e) {
 432                 e.printStackTrace();
 433                 logger.error("Exception in changeCommonNameLabel: " + e.getMessage());
 434             }
 435             }
 436     }
 437
 438     //1. run transmission engine #3979
 439     private void runTransmissionEngine(BerlinModelImportConfigurator config,
 440             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 441
 442         if (doRunTransmissionEngine && (config.getCheck().isImport()  )  ){
 443             try {
 444                 ICdmRepository app = bmImport.getCdmAppController();
 445
 446                 final List<String> term_init_strategy = Arrays.asList(new String []{
 447                         "representations"
 448                 });
 449
 450                 UUID uuidSuperAreaLevel = BerlinModelTransformer.uuidEuroMedAreaLevelFirst;
 451                 NamedAreaLevel euroMedLevel1 = (NamedAreaLevel)app.getTermService().find(uuidSuperAreaLevel);
 452
 453                 Pager<NamedArea> areaPager = app.getTermService().list(
 454                         euroMedLevel1,
 455                         (NamedAreaType) null,
 456                         null,
 457                         null,
 458                         (List<OrderHint>) null,
 459                         term_init_strategy);
 460                 TransmissionEngineDistribution transmissionEngineDistribution = (TransmissionEngineDistribution)app.getBean("transmissionEngineDistribution");
 461                 transmissionEngineDistribution.accumulate(
 462                         AggregationMode.byAreasAndRanks,
 463                         areaPager.getRecords(),
 464                         Rank.UNRANKED_INFRASPECIFIC(),   //or do we even want to start from lower (UNKNOWN?)
 465                         Rank.GENUS(),
 466                         null,
 467                         DefaultProgressMonitor.NewInstance());
 468             } catch (Exception e) {
 469                 e.printStackTrace();
 470                 logger.error("Exception in markAreasAsHidden: " + e.getMessage());
 471             }
 472         }
 473     }
 474
 475 //    //5.Mark areas to be hidden #3979 .5
 476 //    private void markAreasAsHidden(BerlinModelImportConfigurator config,
 477 //            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 478 //
 479 //        if (config.isDoOccurrence() && (config.getCheck().isImport())){
 480 //              try {
 481 //                ICdmRepository app = bmImport.getCdmAppController();
 482 //                TransactionStatus tx = app.startTransaction();
 483 //
 484 //                MarkerType hiddenAreaMarkerType = MarkerType.NewInstance("Used to hide distributions for the named areas in publications", "Hidden Area", null);
 485 //                hiddenAreaMarkerType.setUuid(BerlinModelTransformer.uuidHiddenArea);
 486 //                @SuppressWarnings("unchecked")
 487 //                TermVocabulary<MarkerType> vocUserDefinedMarkerTypes = app.getVocabularyService().find(CdmImportBase.uuidUserDefinedMarkerTypeVocabulary);
 488 //                if (vocUserDefinedMarkerTypes == null){
 489 //                    String message = "Marker type vocabulary could not be found. Hidden areas not added.";
 490 //                    logger.error(message);
 491 //                    System.out.println(message);
 492 //                }else{
 493 //                    vocUserDefinedMarkerTypes.addTerm(hiddenAreaMarkerType);
 494 //                    app.getVocabularyService().saveOrUpdate(vocUserDefinedMarkerTypes);
 495 //
 496 //                    //Add hidden area marker to Rs(C) and Rs(N)
 497 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs);
 498 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_B);
 499 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_C);
 500 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_E);
 501 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_N);
 502 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_K);
 503 //                    hideArea(app, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_W);
 504 //                 }
 505 //                app.commitTransaction(tx);
 506 //            } catch (Exception e) {
 507 //                e.printStackTrace();
 508 //                logger.error("Exception in markAreasAsHidden: " + e.getMessage());
 509 //            }
 510 //          }
 511 //    }
 512 //
 513 //    private void hideArea(ICdmRepository app, MarkerType hiddenAreaMarkerType, UUID areaUuid) {
 514 //        NamedArea area = (NamedArea)app.getTermService().find(areaUuid);
 515 //        area.addMarker(Marker.NewInstance(hiddenAreaMarkerType, true));
 516 //        app.getTermService().saveOrUpdate(area);
 517 //    }
 518
 519     //2. import shapefile attributes #3979 .2
 520     private void importShapefile(BerlinModelImportConfigurator config,
 521             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 522
 523         if (config.isDoOccurrence() && (config.getCheck().isImport())){
 524
 525                try {
 526                    UUID areaVocabularyUuid = BerlinModelTransformer.uuidVocEuroMedAreas;
 527                List<String> idSearchFields = Arrays.asList(new String[]{"EMAREA","PARENT"});
 528                String wmsLayerName = "euromed_2013";
 529                Set<UUID> areaUuidSet = null;
 530
 531                ICdmRepository app = bmImport.getCdmAppController();
 532                IEditGeoService geoService = (IEditGeoService)app.getBean("editGeoService");
 533
 534                Map<NamedArea, String> resultMap;
 535                try {
 536                    InputStream in = CaucasusEuroMedActivator.class.getResourceAsStream("/euromed/euromed_2013.csv");
 537                    Reader reader = new InputStreamReader(in, "UTF-8");
 538
 539                    resultMap = geoService.mapShapeFileToNamedAreas(
 540                                reader, idSearchFields , wmsLayerName , areaVocabularyUuid, areaUuidSet);
 541                    Map<String, String> flatResultMap = new HashMap<>(resultMap.size());
 542                    for(NamedArea area : resultMap.keySet()){
 543                        flatResultMap.put(area.getTitleCache() + " [" + area.getUuid() + "]", resultMap.get(area));
 544                    }
 545                } catch (IOException e) {
 546                     String message = "IOException when reading from mapping file or creating result map.";
 547                     logger.error(message);
 548                     System.out.println(message);
 549                }
 550             } catch (Exception e) {
 551                 e.printStackTrace();
 552                 logger.error("Exception in importShapefile: " + e.getMessage());
 553             }
 554             }
 555     }
 556
 557     //4. Create users and assign roles  #3979
 558     private void createUsersAndRoles(BerlinModelImportConfigurator config,
 559             CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {
 560
 561         try {
 562             if (config.isDoRelTaxa() && (config.getCheck().isImport())){
 563                 ICdmRepository app = bmImport.getCdmAppController();
 564                 TransactionStatus tx = app.startTransaction();
 565
 566                 //eraabstraube
 567                 String eraabstraube = "e.raabstraube";
 568                 List<User> users = app.getUserService().listByUsername(eraabstraube, MatchMode.EXACT, null, null, null, null, null);
 569                 User userEraabStraube;
 570                 if (users.isEmpty()){
 571                     userEraabStraube = User.NewInstance(eraabstraube, eraabstraube);
 572                 }else{
 573                     userEraabStraube = users.get(0);
 574                 }
 575                 if (userEraabStraube.getPerson() == null){
 576                     Person eckhard = Person.NewInstance();
 577                     eckhard.setFamilyName("von Raab-Straube");
 578                     eckhard.setGivenName("Eckhard");
 579                     eckhard.setPrefix("Dr.");
 580                     userEraabStraube.setPerson(eckhard);
 581                 }
 582                 app.getUserService().saveOrUpdate(userEraabStraube);
 583
 584                 //groups
 585                 Group groupEditor = app.getGroupService().load(Group.GROUP_EDITOR_UUID);
 586                 groupEditor.addMember(userEraabStraube);
 587                 app.getGroupService().saveOrUpdate(groupEditor);
 588
 589                 Group groupProjectManager = app.getGroupService().load(Group.GROUP_PROJECT_MANAGER_UUID);
 590                 groupProjectManager.addMember(userEraabStraube);
 591                 app.getGroupService().saveOrUpdate(groupProjectManager);
 592
 593                 String[] publishRoles = new String[]{Role.ROLE_PUBLISH.toString()};
 594                 Group groupPublisher = checkGroup(app.getGroupService(), Group.GROUP_PUBLISHER_UUID, "Publisher", publishRoles);
 595                 groupPublisher.addMember(userEraabStraube);
 596                 app.getGroupService().saveOrUpdate(groupPublisher);
 597
 598                 UUID uuidEuroMedPlantBaseGroup = UUID.fromString("91be42ea-ad04-4458-9836-389277e773db");
 599                 String[] emPlantBaseRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]"};
 600                 Group euroMedPlantbase = checkGroup(app.getGroupService(), uuidEuroMedPlantBaseGroup, "Euro+Med Plantbase", emPlantBaseRoles);
 601                 euroMedPlantbase.addMember(userEraabStraube);
 602                 app.getGroupService().saveOrUpdate(euroMedPlantbase);
 603
 604                 //cichorieae-editor
 605                 String cichorieaeEditor = "cichorieae-editor";
 606                 app.getUserService().listByUsername(cichorieaeEditor, MatchMode.EXACT, null, null, null, null, null);
 607                 User userCichEditor;
 608                 if (users.isEmpty()){
 609                     userCichEditor = User.NewInstance(cichorieaeEditor, cichorieaeEditor);
 610                 }else{
 611                     userCichEditor = users.get(0);
 612                 }
 613                 app.getUserService().saveOrUpdate(userCichEditor);
 614
 615                 //groups
 616                 groupEditor.addMember(userCichEditor);
 617                 app.getGroupService().saveOrUpdate(groupEditor);
 618
 619                 UUID uuidCichorieaeSubtree = null;
 620                 UUID uuidCichorieae = UUID.fromString("63c7dbeb-b9a2-48b8-a75f-e3fe5e161f7c");
 621                 Taxon cich = (Taxon)app.getTaxonService().find(uuidCichorieae);
 622                 if (cich != null){
 623                     if (!cich.getTaxonNodes().isEmpty()){
 624                         TaxonNode cichNode = cich.getTaxonNodes().iterator().next();
 625                         uuidCichorieaeSubtree = cichNode.getUuid();
 626                     }
 627                 }
 628
 629                 String[] cichorieaeRoles = new String[]{};
 630                 if (uuidCichorieaeSubtree != null){
 631                     cichorieaeRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]{"+uuidCichorieaeSubtree.toString()+"}"};
 632                 }else{
 633                     String message = "Cichorieae node could not be found for cichorieae-editor role";
 634                     logger.warn(message);
 635                     System.out.println(message);
 636                 }
 637                 UUID uuidCichorieaeGroup = UUID.fromString("a630938d-dd4f-48c2-9406-91def487b11e");
 638                 String cichorieaeGroupName = "Cichorieae";
 639                 Group cichorieaeGroup = checkGroup(app.getGroupService(), uuidCichorieaeGroup, cichorieaeGroupName, cichorieaeRoles);
 640                 cichorieaeGroup.addMember(userCichEditor);
 641                 app.getGroupService().saveOrUpdate(cichorieaeGroup);
 642
 643                 app.commitTransaction(tx);
 644             }
 645         } catch (Exception e) {
 646             e.printStackTrace();
 647             logger.error("Exception in createUsersAndRoles: " + e.getMessage());
 648         }
 649     }
 650
 651         /**
 652           * copied from {@link FirstDataInserter#checkGroup}
 653      */
 654     private Group checkGroup(IGroupService groupService, UUID groupUuid, String groupName, String[] requiredAuthorities) {
 655         Group group = groupService.load(groupUuid);
 656         if(group == null){
 657             group = Group.NewInstance();
 658             group.setUuid(groupUuid);
 659             logger.info("New Group '" + groupName + "' created");
 660         }
 661         group.setName(groupName); // force name
 662
 663         Set<GrantedAuthority> grantedAuthorities = group.getGrantedAuthorities();
 664
 665         for(String a : requiredAuthorities){
 666             boolean isMissing = true;
 667             for(GrantedAuthority ga : grantedAuthorities){
 668                 if(a.equals(ga.getAuthority())){
 669                     isMissing = false;
 670                     break;
 671                 }
 672             }
 673             if(isMissing){
 674                 GrantedAuthorityImpl newGa = GrantedAuthorityImpl.NewInstance(a);
 675                 group.addGrantedAuthority(newGa);
 676                 logger.info("New GrantedAuthority '" + a + "' added  to '" + groupName + "'");
 677             }
 678         }
 679         groupService.saveOrUpdate(group);
 680         logger.info("Check of group  '" + groupName + "' done");
 681         return group;
 682     }
 683
 684
 685         /**
 686          * @param args
 687          */
 688         public static void main(String[] args) {
 689                 CaucasusEuroMedActivator importActivator = new CaucasusEuroMedActivator();
 690                 Source source = berlinModelSource;
 691                 ICdmDataSource cdmRepository = CdmDestinations.chooseDestination(args) != null ? CdmDestinations.chooseDestination(args) : cdmDestination;
 692
 693                 importActivator.importEm2CDM(source, cdmRepository, hbm2dll);
 694                 if (includePesiExport){
 695                         //not available from here since E+M was moved to app-import
 696 //                      PesiExportActivatorEM exportActivator = new PesiExportActivatorEM();
 697 //                      exportActivator.doExport(cdmRepository);
 698                 }
 699                 System.exit(0);
 700
 701         }
 702
 703 }