EDIT

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.app.berlinModelImport;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.UUID;

import org.apache.log4j.Logger;

import org.springframework.security.core.GrantedAuthority;

import org.springframework.transaction.TransactionStatus;

import eu.etaxonomy.cdm.api.application.FirstDataInserter;

import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;

import eu.etaxonomy.cdm.api.service.IGroupService;

import eu.etaxonomy.cdm.api.service.description.TransmissionEngineDistribution;

import eu.etaxonomy.cdm.api.service.description.TransmissionEngineDistribution.AggregationMode;

import eu.etaxonomy.cdm.api.service.pager.Pager;

import eu.etaxonomy.cdm.app.common.CdmDestinations;

import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;

import eu.etaxonomy.cdm.database.DbSchemaValidation;

import eu.etaxonomy.cdm.database.ICdmDataSource;

import eu.etaxonomy.cdm.ext.geo.IEditGeoService;

import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;

import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;

import eu.etaxonomy.cdm.io.common.CdmDefaultImport;

import eu.etaxonomy.cdm.io.common.CdmImportBase;

import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;

import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;

import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;

import eu.etaxonomy.cdm.io.common.Source;

import eu.etaxonomy.cdm.model.agent.Person;

import eu.etaxonomy.cdm.model.common.DefinedTermBase;

import eu.etaxonomy.cdm.model.common.GrantedAuthorityImpl;

import eu.etaxonomy.cdm.model.common.Group;

import eu.etaxonomy.cdm.model.common.Language;

import eu.etaxonomy.cdm.model.common.Marker;

import eu.etaxonomy.cdm.model.common.MarkerType;

import eu.etaxonomy.cdm.model.common.Representation;

import eu.etaxonomy.cdm.model.common.TermVocabulary;

import eu.etaxonomy.cdm.model.common.User;

import eu.etaxonomy.cdm.model.description.Feature;

import eu.etaxonomy.cdm.model.description.FeatureNode;

import eu.etaxonomy.cdm.model.description.FeatureTree;

import eu.etaxonomy.cdm.model.location.NamedArea;

import eu.etaxonomy.cdm.model.location.NamedAreaLevel;

import eu.etaxonomy.cdm.model.location.NamedAreaType;

import eu.etaxonomy.cdm.model.name.NomenclaturalCode;

import eu.etaxonomy.cdm.model.name.Rank;

import eu.etaxonomy.cdm.model.taxon.Taxon;

import eu.etaxonomy.cdm.model.taxon.TaxonNode;

import eu.etaxonomy.cdm.persistence.hibernate.permission.Role;

import eu.etaxonomy.cdm.persistence.query.MatchMode;

import eu.etaxonomy.cdm.persistence.query.OrderHint;

/**

 * TODO add the following to a wiki page:

 * HINT: If you are about to import into a mysql data base running under windows and if you wish to dump and restore the resulting data bas under another operation systen

 * you must set the mysql system variable lower_case_table_names = 0 in order to create data base with table compatible names.

 *

 *

 * @author a.mueller

 *

 */

public class EuroMedActivator {

	private static final Logger logger = Logger.getLogger(EuroMedActivator.class);

	//database validation status (create, update, validate ...)

	static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;

	static final Source berlinModelSource = BerlinModelSources.euroMed_BGBM42();

//	static final Source berlinModelSource = BerlinModelSources.euroMed_PESI3();

	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_euroMed();

//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_pesi_euromed();

//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_euromed3();

//	static final ICdmDataSource cdmDestination = CdmDestinations.localH2();

	static final boolean includePesiExport = false;

	static final int sourceSecId = 7000000; //500000

	static final UUID classificationUuid = UUID.fromString("314a68f9-8449-495a-91c2-92fde8bcf344");

	static final boolean useSingleClassification = true;

	static final String classificationName = "Euro+Med 2014";

	static final UUID featureTreeUuid = UUID.fromString("6a5e1c2b-ec0d-46c8-9c7d-a2059267ffb7");

	static final Object[] featureKeyList = new Integer[]{1, 31, 4, 98, 41};

	// set to zero for unlimited nameFacts

	static final int maximumNumberOfNameFacts = 0;

	static final int partitionSize = 2500;

	//check - import

	static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;

	//editor - import

	static final EDITOR editor = EDITOR.EDITOR_AS_EDITOR;

	//NomenclaturalCode

	static final NomenclaturalCode nomenclaturalCode = NomenclaturalCode.ICNAFP;

	//ignore null

	static final boolean ignoreNull = true;

	static final boolean switchSpeciesGroup = true;

	static boolean useClassification = true;

	static boolean isSplitTdwgCodes = false;

	static boolean useEmAreaVocabulary = true;

	private final boolean removeHttpMapsAnchor = true;

	static final String infrGenericRankAbbrev = "[unranked]";

	static final String infrSpecificRankAbbrev = "[unranked]";

	static String taxonTable = "v_cdm_exp_taxaAll";

	static String classificationQuery = " SELECT DISTINCT t.PTRefFk, r.RefCache FROM PTaxon t INNER JOIN Reference r ON t.PTRefFk = r.RefId WHERE t.PTRefFk = " + sourceSecId;

	static String relPTaxonIdQuery = " SELECT r.RelPTaxonId " +

					" FROM RelPTaxon AS r INNER JOIN v_cdm_exp_taxaDirect AS a ON r.PTNameFk2 = a.PTNameFk AND r.PTRefFk2 = a.PTRefFk" +

					" WHERE r.RelPTaxonID NOT IN (1883791,1883800,1883823,1883852,1883853,1883864,1883896,1883921,1883943,1883944,1883955,1883972,1884008,1884030,1884031,1884035) " + //Unpublished Kew taxa which are not imported anymore

					" ";  // AND r.RelQualifierFk =1

	static String nameIdTable = " v_cdm_exp_namesAll ";

	static String referenceIdTable = " v_cdm_exp_refAll ";

	static String refDetailFilter =  " RefDetailID IN (SELECT RefDetailID FROM v_cdm_exp_RefDetail) ";

	static String factFilter = " factId IN ( SELECT factId FROM v_cdm_exp_factsAll WHERE FactCategoryFk NOT IN (12, 14, 249, 251))";

	static String occurrenceFilter = " occurrenceId IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";

	static String occurrenceSourceFilter = " occurrenceFk IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";

	static String commonNameFilter = " commonNameId IN ( SELECT commonNameId FROM v_cdm_exp_commonNamesAll )";

	static String webMarkerFilter = " TableNameFk <> 500 OR ( RIdentifierFk IN (SELECT RIdentifier FROM v_cdm_exp_taxaAll)) ";

	static String authorTeamFilter = null; // " authorTeamId IN (SELECT authorTeamId FROM v_cdm_exp_authorTeamsAll) ";

	static String authorFilter = null;  // " authorId IN (SELECT authorId FROM v_cdm_exp_authorsAll) ";

// **************** ALL *********************

	boolean invers = !(hbm2dll == DbSchemaValidation.CREATE);

	static final boolean doUser = true;

//	//authors

	static final boolean doAuthors = true;

	//references

	static final DO_REFERENCES doReferences =  DO_REFERENCES.ALL;

	//names

	static final boolean doTaxonNames = true;

	static final boolean doRelNames = true;

	static final boolean doNameStatus = true;

	static final boolean doTypes = false;  //serious types do not exist in E+M

	static final boolean doNameFacts = true;

	//taxa

	static final boolean doTaxa = true;

	static final boolean doRelTaxa = true;  //FIXME revert

	static final boolean doFacts = true;

	static final boolean doOccurences = true;

	static final boolean doCommonNames = true;

	//etc.

	static final boolean doMarker = true;

	public void importEm2CDM (Source source, ICdmDataSource destination, DbSchemaValidation hbm2dll){

		System.out.println("Start import from BerlinModel("+ berlinModelSource.getDatabase() + ") to " + cdmDestination.getDatabase() + " ...");

		//make BerlinModel Source

		BerlinModelImportConfigurator config = BerlinModelImportConfigurator.NewInstance(source,  destination);

		config.setClassificationName(classificationName);

		config.setClassificationUuid(classificationUuid);

		config.setSourceSecId(sourceSecId);

		config.setNomenclaturalCode(nomenclaturalCode);

		config.setIgnoreNull(ignoreNull);

		config.setDoAuthors(doAuthors ^ invers);

		config.setDoReferences(invers ? doReferences.invers() : doReferences);

		config.setDoTaxonNames(doTaxonNames ^ invers);

		config.setDoRelNames(doRelNames ^ invers);

		config.setDoNameStatus(doNameStatus ^ invers);

		config.setDoTypes(doTypes);  //always false

		config.setDoNameFacts(doNameFacts ^ invers);

		config.setDoTaxa(doTaxa ^ invers);

		config.setDoRelTaxa(doRelTaxa ^ invers);

		config.setDoFacts(doFacts ^ invers);

		config.setDoOccurrence(doOccurences ^ invers);

		config.setDoCommonNames(doCommonNames ^ invers);

		config.setDoMarker(doMarker ^ invers);

		config.setDoUser(doUser ^ invers);

		config.setUseClassification(useClassification);

		config.setSourceRefUuid(BerlinModelTransformer.uuidSourceRefEuroMed);

		config.setEditor(editor);

		config.setDbSchemaValidation(hbm2dll);

		// maximum number of name facts to import

		config.setMaximumNumberOfNameFacts(maximumNumberOfNameFacts);

		config.setInfrGenericRankAbbrev(infrGenericRankAbbrev);

		config.setInfrSpecificRankAbbrev(infrSpecificRankAbbrev);

		config.setRemoveHttpMapsAnchor(removeHttpMapsAnchor);

//		filter

		config.setTaxonTable(taxonTable);

		config.setClassificationQuery(classificationQuery);

		config.setRelTaxaIdQuery(relPTaxonIdQuery);

		config.setNameIdTable(nameIdTable);

		config.setReferenceIdTable(referenceIdTable);

		config.setAuthorTeamFilter(authorTeamFilter);

		config.setAuthorFilter(authorFilter);

		config.setFactFilter(factFilter);

		config.setRefDetailFilter(refDetailFilter);

		config.setCommonNameFilter(commonNameFilter);

		config.setOccurrenceFilter(occurrenceFilter);

		config.setOccurrenceSourceFilter(occurrenceSourceFilter);

		config.setWebMarkerFilter(webMarkerFilter);

		config.setUseSingleClassification(useSingleClassification);

		//TDWG codes

		config.setSplitTdwgCodes(isSplitTdwgCodes);

		config.setUseEmAreaVocabulary(useEmAreaVocabulary);

		config.setCheck(check);

		config.setEditor(editor);

		config.setRecordsPerTransaction(partitionSize);

		config.setSwitchSpeciesGroup(switchSpeciesGroup);

		// invoke import

		CdmDefaultImport<BerlinModelImportConfigurator> bmImport = new CdmDefaultImport<BerlinModelImportConfigurator>();

		bmImport.invoke(config);

		renameRanks(config, bmImport);

		createFeatureTree(config, bmImport);

		changeCommonNameLabel(config, bmImport);

        runTransmissionEngine(config, bmImport);

        importShapefile(config, bmImport);

        createUsersAndRoles(config, bmImport);

	    markAreasAsHidden(config, bmImport);

		System.out.println("End import from BerlinModel ("+ source.getDatabase() + ")...");

	}

    private void renameRanks(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        //Rename Ranks (still needed?)

		if (config.isDoTaxonNames() && (config.getCheck().isImport() )  ){

			ICdmApplicationConfiguration app = bmImport.getCdmAppController();

			TransactionStatus tx = app.startTransaction();

			try {

				Rank sectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());

				Representation repr = sectBot.getRepresentation(Language.ENGLISH());

				repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());

				repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());

				sectBot.setTitleCache(null, false);  //to definetely update the titleCache also

				app.getTermService().saveOrUpdate(sectBot);

				Rank subSectBot = (Rank)app.getTermService().find(Rank.SECTION_BOTANY().getUuid());

				repr = subSectBot.getRepresentation(Language.ENGLISH());

				repr.setAbbreviatedLabel(repr.getAbbreviatedLabel().replace("(bot.)", "").trim());

				repr.setLabel(repr.getLabel().replace("(Botany)", "").trim());

				subSectBot.setTitleCache(null, false);  //to definetely update the titleCache also

				app.getTermService().saveOrUpdate(subSectBot);

				app.commitTransaction(tx);

			} catch (Exception e) {

				e.printStackTrace();

			}

		}

    }

    private void createFeatureTree(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        //create feature tree

		if (config.isDoFacts() && (config.getCheck().isImport()  )  ){

			ICdmApplicationConfiguration app = bmImport.getCdmAppController();

			TransactionStatus tx = app.startTransaction();

			//make feature tree

			FeatureTree tree = TreeCreator.flatTree(featureTreeUuid, config.getFeatureMap(), featureKeyList);

			FeatureNode imageNode = FeatureNode.NewInstance(Feature.IMAGE());

			tree.getRoot().addChild(imageNode);

			FeatureNode distributionNode = FeatureNode.NewInstance(Feature.DISTRIBUTION());

			tree.getRoot().addChild(distributionNode, 1);

			FeatureNode commonNameNode = FeatureNode.NewInstance(Feature.COMMON_NAME());

			tree.getRoot().addChild(commonNameNode, 2);

			app.getFeatureTreeService().saveOrUpdate(tree);

            app.commitTransaction(tx);

		}

    }

    private void changeCommonNameLabel(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        //Change common name label

	    if (config.isDoFacts() && (config.getCheck().isImport()  )  ){

	        ICdmApplicationConfiguration app = bmImport.getCdmAppController();

	        TransactionStatus tx = app.startTransaction();

	        DefinedTermBase<?> commonNameFeature = app.getTermService().find(Feature.COMMON_NAME().getUuid());

            commonNameFeature.setLabel("Common Names", Language.ENGLISH());

            commonNameFeature.setTitleCache(null, false);  //to definitely update the titleCache also

            app.getTermService().saveOrUpdate(commonNameFeature);

            app.commitTransaction(tx);

	    }

    }

    //1. run transmission engine #3979

    private void runTransmissionEngine(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        //Transmission engine #3979 .1

        if (true || config.isDoOccurrence() && (config.getCheck().isImport()  )  ){

            ICdmApplicationConfiguration app = bmImport.getCdmAppController();

            final List<String> term_init_strategy = Arrays.asList(new String []{

                    "representations"

            });

            UUID uuidSuperAreaLevel = BerlinModelTransformer.uuidEuroMedAreaLevelFirst;

            NamedAreaLevel euroMedLevel1 = (NamedAreaLevel)app.getTermService().find(uuidSuperAreaLevel);

            Pager<NamedArea> areaPager = app.getTermService().list(

                    euroMedLevel1,

                    (NamedAreaType) null,

                    null,

                    null,

                    (List<OrderHint>) null,

                    term_init_strategy);

            TransmissionEngineDistribution transmissionEngineDistribution = (TransmissionEngineDistribution)app.getBean("transmissionEngineDistribution");

            transmissionEngineDistribution.accumulate(

                    AggregationMode.byAreasAndRanks,

                    areaPager.getRecords(),

                    Rank.SUBSPECIES(),

                    Rank.GENUS(),

                    null,

                    DefaultProgressMonitor.NewInstance());

        }

    }

    //5.Mark areas to be hidden #3979 .5

    private void markAreasAsHidden(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        if (config.isDoOccurrence() && (config.getCheck().isImport())){

	        ICdmApplicationConfiguration app = bmImport.getCdmAppController();

	        TransactionStatus tx = app.startTransaction();

	        MarkerType hiddenAreaMarkerType = MarkerType.NewInstance("Used to hide distributions for the named areas in publications", "Hidden Area", null);

	        hiddenAreaMarkerType.setUuid(BerlinModelTransformer.uuidHiddenArea);

	        @SuppressWarnings("unchecked")

            TermVocabulary<MarkerType> vocUserDefinedMarkerTypes = app.getVocabularyService().find(CdmImportBase.uuidUserDefinedMarkerTypeVocabulary);

	        if (vocUserDefinedMarkerTypes == null){

	            String message = "Marker type vocabulary could not be found. Hidden areas not added.";

	            logger.error(message);

	            System.out.println(message);

	        }else{

	            vocUserDefinedMarkerTypes.addTerm(hiddenAreaMarkerType);

	            app.getVocabularyService().saveOrUpdate(vocUserDefinedMarkerTypes);

	            //Add hidden area marker to Rs(C) and Rs(N)

	            NamedArea rs_c = (NamedArea)app.getTermService().find(BerlinModelTransformer.uuidRs_C);

	            rs_c.addMarker(Marker.NewInstance(hiddenAreaMarkerType, true));

	            app.getTermService().saveOrUpdate(rs_c);

	            NamedArea rs_n = (NamedArea)app.getTermService().find(BerlinModelTransformer.uuidRs_N);

                rs_n.addMarker(Marker.NewInstance(hiddenAreaMarkerType, true));

                app.getTermService().saveOrUpdate(rs_n);

	        }

	        app.commitTransaction(tx);

	    }

    }

    //2. import shapefile attributes #3979 .2

    private void importShapefile(BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        if (config.isDoOccurrence() && (config.getCheck().isImport())){

	       UUID areaVocabularyUuid = BerlinModelTransformer.uuidVocEuroMedAreas;

           List<String> idSearchFields = Arrays.asList(new String[]{"EMAREA","PARENT"});

           String wmsLayerName = "euromed_2013";

           Set<UUID> areaUuidSet = null;

           ICdmApplicationConfiguration app = bmImport.getCdmAppController();

           IEditGeoService geoService = (IEditGeoService)app.getBean("editGeoService");

           Map<NamedArea, String> resultMap;

           try {

//             URL url =  ClassLoader.getSystemResource("myConfig.txt");

//             FileReader fileReader = new FileReader(url.getFile());

               InputStream in = EuroMedActivator.class.getResourceAsStream("/euromed/euromed_2013.csv");

               Reader reader = new InputStreamReader(in, "UTF-8");

               resultMap = geoService.mapShapeFileToNamedAreas(

                           reader, idSearchFields , wmsLayerName , areaVocabularyUuid, areaUuidSet);

               Map<String, String> flatResultMap = new HashMap<String, String>(resultMap.size());

               for(NamedArea area : resultMap.keySet()){

                   flatResultMap.put(area.getTitleCache() + " [" + area.getUuid() + "]", resultMap.get(area));

               }

           } catch (IOException e) {

                String message = "IOException when reading from mapping file or creating result map.";

                logger.error(message);

                System.out.println(message);

           }

	    }

    }

    //4. Create users and assign roles  #3979

    private void createUsersAndRoles(

            BerlinModelImportConfigurator config,

            CdmDefaultImport<BerlinModelImportConfigurator> bmImport) {

        if (config.isDoRelTaxa() && (config.getCheck().isImport())){

	        ICdmApplicationConfiguration app = bmImport.getCdmAppController();

	        TransactionStatus tx = app.startTransaction();

	        //eraabstraube

	        String eraabstraube = "eraabstraube";

	        List<User> users = app.getUserService().listByUsername(eraabstraube, MatchMode.EXACT, null, null, null, null, null);

	        User userEraabStraube;

	        if (users.isEmpty()){

	            userEraabStraube = User.NewInstance(eraabstraube, eraabstraube);

	        }else{

	            userEraabStraube = users.get(0);

	        }

	        if (userEraabStraube.getPerson() == null){

    	        Person eckhard = Person.NewInstance();

                eckhard.setLastname("von Raab-Straube");

                eckhard.setFirstname("Eckhard");

                eckhard.setPrefix("Dr.");

                userEraabStraube.setPerson(eckhard);

	        }

            app.getUserService().saveOrUpdate(userEraabStraube);

            //groups

            Group groupEditor = app.getGroupService().load(Group.groupEditorUuid);

            groupEditor.addMember(userEraabStraube);

            app.getGroupService().saveOrUpdate(groupEditor);

            Group groupProjectManager = app.getGroupService().load(Group.groupProjectManagerUuid);

            groupProjectManager.addMember(userEraabStraube);

            app.getGroupService().saveOrUpdate(groupProjectManager);

            String[] publishRoles = new String[]{Role.ROLE_PUBLISH.toString()};

            Group groupPublisher = checkGroup(app.getGroupService(), Group.groupPublisherUuid, "Publisher", publishRoles);

            groupPublisher.addMember(userEraabStraube);

            app.getGroupService().saveOrUpdate(groupPublisher);

            UUID uuidEuroMedPlantBaseGroup = UUID.fromString("91be42ea-ad04-4458-9836-389277e773db");

            String[] emPlantBaseRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]"};

            Group euroMedPlantbase = checkGroup(app.getGroupService(), uuidEuroMedPlantBaseGroup, "Euro+Med Plantbase", emPlantBaseRoles);

            euroMedPlantbase.addMember(userEraabStraube);

            app.getGroupService().saveOrUpdate(euroMedPlantbase);

            //cichorieae-editor

            String cichorieaeEditor = "cichorieae-editor";

            app.getUserService().listByUsername(cichorieaeEditor, MatchMode.EXACT, null, null, null, null, null);

            User userCichEditor;

            if (users.isEmpty()){

                userCichEditor = User.NewInstance(cichorieaeEditor, cichorieaeEditor);

            }else{

                userCichEditor = users.get(0);

            }

            app.getUserService().saveOrUpdate(userCichEditor);

            //groups

            groupEditor.addMember(userCichEditor);

            app.getGroupService().saveOrUpdate(groupEditor);

            UUID uuidCichorieaeSubtree = null;

            UUID uuidCichorieae = UUID.fromString("63c7dbeb-b9a2-48b8-a75f-e3fe5e161f7c");

            Taxon cich = (Taxon)app.getTaxonService().find(uuidCichorieae);

            if (cich != null){

                TaxonNode cichNode = cich.getTaxonNodes().iterator().next();

                if (cichNode != null){

                    uuidCichorieaeSubtree = cichNode.getUuid();

                }

            }

            String[] cichorieaeRoles = new String[]{};

            if (uuidCichorieaeSubtree != null){

                cichorieaeRoles = new String[]{"TAXONNODE.[CREATE,READ,UPDATE,DELETE]{"+uuidCichorieaeSubtree.toString()+"}"};

            }else{

                String message = "Cichorieae node could not be found";

                logger.warn(message);

                System.out.println(message);

            }

            UUID uuidCichorieaeGroup = UUID.fromString("a630938d-dd4f-48c2-9406-91def487b11e");

            String cichorieaeGroupName = "Cichorieae";

            Group cichorieaeGroup = checkGroup(app.getGroupService(), uuidCichorieaeGroup, cichorieaeGroupName, cichorieaeRoles);

            cichorieaeGroup.addMember(userCichEditor);

            app.getGroupService().saveOrUpdate(cichorieaeGroup);

            app.commitTransaction(tx);

	    }

    }

	/**

	  * copied from {@link FirstDataInserter#checkGroup}

     */

    private Group checkGroup(IGroupService groupService, UUID groupUuid, String groupName, String[] requiredAuthorities) {

        Group group = groupService.load(groupUuid);

        if(group == null){

            group = Group.NewInstance();

            group.setUuid(groupUuid);

            logger.info("New Group '" + groupName + "' created");

        }

        group.setName(groupName); // force name

        Set<GrantedAuthority> grantedAuthorities = group.getGrantedAuthorities();

        for(String a : requiredAuthorities){

            boolean isMissing = true;

            for(GrantedAuthority ga : grantedAuthorities){

                if(a.equals(ga.getAuthority())){

                    isMissing = false;

                    break;

                }

            }

            if(isMissing){

                GrantedAuthorityImpl newGa = GrantedAuthorityImpl.NewInstance();

                newGa.setAuthority(a);

                group.addGrantedAuthority(newGa);

                logger.info("New GrantedAuthority '" + a + "' added  to '" + groupName + "'");

            }

        }

        groupService.saveOrUpdate(group);

        logger.info("Check of group  '" + groupName + "' done");

        return group;

    }

	/**

	 * @param args

	 */

	public static void main(String[] args) {

		EuroMedActivator importActivator = new EuroMedActivator();

		Source source = berlinModelSource;

		ICdmDataSource cdmRepository = CdmDestinations.chooseDestination(args) != null ? CdmDestinations.chooseDestination(args) : cdmDestination;

		importActivator.importEm2CDM(source, cdmRepository, hbm2dll);

		if (includePesiExport){

			//not available from here since E+M was moved to app-import

//			PesiExportActivatorEM exportActivator = new PesiExportActivatorEM();

//			exportActivator.doExport(cdmRepository);

		}

	}

}

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.io.berlinModel.in;

import java.net.URI;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.UUID;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;

import org.springframework.stereotype.Component;

import org.springframework.transaction.TransactionStatus;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;

import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;

import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelOccurrenceImportValidator;

import eu.etaxonomy.cdm.io.common.IOValidator;

import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;

import eu.etaxonomy.cdm.io.common.Source;

import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;

import eu.etaxonomy.cdm.model.common.Annotation;

import eu.etaxonomy.cdm.model.common.AnnotationType;

import eu.etaxonomy.cdm.model.common.CdmBase;

import eu.etaxonomy.cdm.model.common.ExtensionType;

import eu.etaxonomy.cdm.model.common.Language;

import eu.etaxonomy.cdm.model.common.Marker;

import eu.etaxonomy.cdm.model.common.MarkerType;

import eu.etaxonomy.cdm.model.common.OriginalSourceType;

import eu.etaxonomy.cdm.model.common.TermType;

import eu.etaxonomy.cdm.model.common.TermVocabulary;

import eu.etaxonomy.cdm.model.description.DescriptionElementBase;

import eu.etaxonomy.cdm.model.description.Distribution;

import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;

import eu.etaxonomy.cdm.model.description.TaxonDescription;

import eu.etaxonomy.cdm.model.location.NamedArea;

import eu.etaxonomy.cdm.model.location.NamedAreaLevel;

import eu.etaxonomy.cdm.model.location.NamedAreaType;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.taxon.Taxon;

import eu.etaxonomy.cdm.model.taxon.TaxonBase;

import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;

/**

 * @author a.mueller

 * @created 20.03.2008

 */

@Component

public class BerlinModelOccurrenceImport  extends BerlinModelImportBase {

	private static final String EM_AREA_NAMESPACE = "emArea";

	private static final Logger logger = Logger.getLogger(BerlinModelOccurrenceImport.class);

	public static final String NAMESPACE = "Occurrence";

	private static int modCount = 5000;

	private static final String pluralString = "occurrences";

	private static final String dbTableName = "emOccurrence";  //??

	public BerlinModelOccurrenceImport(){

		super(dbTableName, pluralString);

	}

	@Override

	protected String getIdQuery(BerlinModelImportState state) {

		String result = " SELECT occurrenceId FROM " + getTableName();

		if (StringUtils.isNotBlank(state.getConfig().getOccurrenceFilter())){

			result += " WHERE " +  state.getConfig().getOccurrenceFilter();

		}

		return result;

	}

	@Override

	protected String getRecordQuery(BerlinModelImportConfigurator config) {

			String emCode = config.isIncludesAreaEmCode()? ", emArea.EMCode" : "";

			String strQuery =   //DISTINCT because otherwise emOccurrenceSource creates multiple records for a single distribution

            " SELECT DISTINCT PTaxon.RIdentifier AS taxonId, emOccurrence.OccurrenceId, emOccurrence.Native, emOccurrence.Introduced, " +

            		" emOccurrence.Cultivated, emOccurrence.Notes occNotes, " +

            		" emOccurSumCat.emOccurSumCatId, emOccurSumCat.Short, emOccurSumCat.Description, " +

                	" emOccurSumCat.OutputCode, emArea.AreaId, emArea.TDWGCode " + emCode +

                " FROM emOccurrence INNER JOIN " +

                	" emArea ON emOccurrence.AreaFk = emArea.AreaId INNER JOIN " +

                	" PTaxon ON emOccurrence.PTNameFk = PTaxon.PTNameFk AND emOccurrence.PTRefFk = PTaxon.PTRefFk LEFT OUTER JOIN " +

                	" emOccurSumCat ON emOccurrence.SummaryStatus = emOccurSumCat.emOccurSumCatId LEFT OUTER JOIN " +

                	" emOccurrenceSource ON emOccurrence.OccurrenceId = emOccurrenceSource.OccurrenceFk " +

            " WHERE (emOccurrence.OccurrenceId IN (" + ID_LIST_TOKEN + ")  )" +

                " ORDER BY PTaxon.RIdentifier";

		return strQuery;

	}

	private Map<Integer, NamedArea> euroMedAreas = new HashMap<Integer, NamedArea>();

	@Override

	public void doInvoke(BerlinModelImportState state) {

		if (state.getConfig().isUseEmAreaVocabulary()){

			try {

				createEuroMedAreas(state);

			} catch (Exception e) {

				logger.error("Exception occurred when trying to create euroMed Areas");

				e.printStackTrace();

				state.setSuccess(false);

			}

		}

		super.doInvoke(state);

		//reset

		euroMedAreas = new HashMap<Integer, NamedArea>();

	}

	private TermVocabulary<NamedArea> createEuroMedAreas(BerlinModelImportState state) throws SQLException {

		logger.warn("Start creating E+M areas");

		Source source = state.getConfig().getSource();

		Reference<?> sourceReference = state.getConfig().getSourceReference();

		TransactionStatus txStatus = this.startTransaction();

		sourceReference = getSourceReference(sourceReference);

		TermVocabulary<NamedArea> euroMedAreas = makeEmptyEuroMedVocabulary();

		MarkerType eurMarkerType = getMarkerType(state, BerlinModelTransformer.uuidEurArea, "eur", "eur Area", "eur");

		MarkerType euroMedAreaMarkerType = getMarkerType(state, BerlinModelTransformer.uuidEurMedArea, "EuroMedArea", "EuroMedArea", "EuroMedArea");

		ExtensionType isoCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidIsoCode, "IsoCode", "IsoCode", "iso");

		ExtensionType tdwgCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidTdwgAreaCode, "TDWG code", "TDWG Area code", "tdwg");

		ExtensionType mclCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidMclCode, "MCL code", "MedCheckList code", "mcl");

		NamedAreaLevel areaLevelTop = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelTop, "Euro+Med top area level", "Euro+Med top area level. This level is only to be used for the area representing the complete Euro+Med area", "e+m top", null);

		NamedAreaLevel areaLevelEm1 = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelFirst, "Euro+Med 1. area level", "Euro+Med 1. area level", "e+m 1.", null);

		NamedAreaLevel areaLevelEm2 = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelSecond, "Euro+Med 2. area level", "Euro+Med 2. area level", "Euro+Med 1. area level", null);

		String sql = "SELECT * , CASE WHEN EMCode = 'EM' THEN 'a' ELSE 'b' END as isEM " +

				" FROM emArea " +

				" ORDER BY isEM, EMCode";

		ResultSet rs = source.getResultSet(sql);

		NamedArea euroMedArea = null;

		NamedArea lastLevel2Area = null;

		//euroMedArea (EMCode = 'EM')

		rs.next();

		euroMedArea = makeSingleEuroMedArea(rs, eurMarkerType, euroMedAreaMarkerType, isoCodeExtType, tdwgCodeExtType, mclCodeExtType,

				areaLevelTop, areaLevelEm1 , areaLevelEm2, sourceReference, euroMedArea, lastLevel2Area);

		euroMedAreas.addTerm(euroMedArea);

		//all other areas

		while (rs.next()){

			NamedArea newArea = makeSingleEuroMedArea(rs, eurMarkerType, euroMedAreaMarkerType,

					isoCodeExtType, tdwgCodeExtType, mclCodeExtType,

					areaLevelTop, areaLevelEm1 , areaLevelEm2, sourceReference, euroMedArea, lastLevel2Area);

			euroMedAreas.addTerm(newArea);

			if (newArea.getPartOf().equals(euroMedArea)){

				lastLevel2Area = newArea;

			}

			getVocabularyService().saveOrUpdate(euroMedAreas);

		}

		commitTransaction(txStatus);

		logger.warn("Created E+M areas");

		return euroMedAreas;

	}

	/**

	 * @param sourceReference

	 * @return

	 */

	private Reference<?> getSourceReference(Reference<?> sourceReference) {

		Reference<?> persistentSourceReference = getReferenceService().find(sourceReference.getUuid());  //just to be sure

		if (persistentSourceReference != null){

			sourceReference = persistentSourceReference;

		}

		return sourceReference;

	}

	/**

	 * @param eurMarkerType

	 * @param euroMedAreaMarkerType

	 * @param isoCodeExtType

	 * @param tdwgCodeExtType

	 * @param mclCodeExtType

	 * @param rs

	 * @param areaLevelEm2

	 * @param areaLevelEm1

	 * @param areaLevelTop

	 * @throws SQLException

	 */

	private NamedArea makeSingleEuroMedArea(ResultSet rs, MarkerType eurMarkerType,

			MarkerType euroMedAreaMarkerType, ExtensionType isoCodeExtType,

			ExtensionType tdwgCodeExtType, ExtensionType mclCodeExtType,

			NamedAreaLevel areaLevelTop, NamedAreaLevel areaLevelEm1, NamedAreaLevel areaLevelEm2,

			Reference<?> sourceReference, NamedArea euroMedArea, NamedArea level2Area) throws SQLException {

		Integer areaId = rs.getInt("AreaId");

		String emCode = nullSafeTrim(rs.getString("EMCode"));

		String isoCode = nullSafeTrim(rs.getString("ISOCode"));

		String tdwgCode = nullSafeTrim(rs.getString("TDWGCode"));

		String unit = nullSafeTrim(rs.getString("Unit"));

//				      ,[Status]

//				      ,[OutputOrder]

		boolean eurMarker = rs.getBoolean("eur");

		boolean euroMedAreaMarker = rs.getBoolean("EuroMedArea");

		String notes = nullSafeTrim(rs.getString("Notes"));

		String mclCode = nullSafeTrim(rs.getString("MCLCode"));

		String geoSearch = nullSafeTrim(rs.getString("NameForGeoSearch"));

		if (isBlank(emCode)){

			emCode = unit;

		}

		//uuid

		UUID uuid = BerlinModelTransformer.getEMAreaUuid(emCode);

		NamedArea area = (NamedArea)getTermService().find(uuid);

		if (area == null){

			//label

			area = NamedArea.NewInstance(geoSearch, unit, emCode);

			if (uuid != null){

				area.setUuid(uuid);

			}else{

				logger.warn("Uuuid for emCode could not be defined: " + emCode);

			}

		}

		//code

		area.setIdInVocabulary(emCode);

		//notes

		if (StringUtils.isNotEmpty(notes)){

			area.addAnnotation(Annotation.NewInstance(notes, AnnotationType.EDITORIAL(), Language.DEFAULT()));

		}

		//markers

		area.addMarker(Marker.NewInstance(eurMarkerType, eurMarker));

		area.addMarker(Marker.NewInstance(euroMedAreaMarkerType, euroMedAreaMarker));

		//extensions

		if (isNotBlank(isoCode)){

			area.addExtension(isoCode, isoCodeExtType);

		}

		if (isNotBlank(tdwgCode)){

			area.addExtension(tdwgCode, tdwgCodeExtType);

		}

		if (isNotBlank(mclCode)){

			area.addExtension(mclCode, mclCodeExtType);

		}

		//type

		area.setType(NamedAreaType.ADMINISTRATION_AREA());

		//source

		area.addSource(OriginalSourceType.Import, String.valueOf(areaId), EM_AREA_NAMESPACE, sourceReference, null);

		//parent

		if (euroMedArea != null){

			if (emCode.contains("(")){

				area.setPartOf(level2Area);

				area.setLevel(areaLevelEm2);

			}else{

				area.setPartOf(euroMedArea);

				area.setLevel(areaLevelEm1);

			}

		}else{

			area.setLevel(areaLevelTop);

		}

		this.euroMedAreas.put(areaId, area);

		//save

		getTermService().saveOrUpdate(area);

		return area;

	}

	private String nullSafeTrim(String string) {

		if (string == null){

			return null;

		}else{

			return string.trim();

		}

	}

	/**

	 *

	 */

	private TermVocabulary<NamedArea> makeEmptyEuroMedVocabulary() {

		TermType type = TermType.NamedArea;

		String description = "Euro+Med area vocabulary";

		String label = "E+M areas";

		String abbrev = null;

		URI termSourceUri = null;

		TermVocabulary<NamedArea> result = TermVocabulary.NewInstance(type, description, label, abbrev, termSourceUri);

		result.setUuid(BerlinModelTransformer.uuidVocEuroMedAreas);

		getVocabularyService().save(result);

		return result;

	}

	@Override

	public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {

		boolean success = true;

		Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();

		Map<String, TaxonBase<?>> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);

		ResultSet rs = partitioner.getResultSet();

		try {

			//map to store the mapping of duplicate berlin model occurrences to their real distributions

			//duplicated may occur due to area mappings from BM areas to TDWG areas

			Map<Integer, String> duplicateMap = new HashMap<Integer, String>();

			int oldTaxonId = -1;

			TaxonDescription oldDescription = null;

			int i = 0;

			int countDescriptions = 0;

			int countDistributions = 0;

			int countDuplicates = 0;

			//for each reference

            while (rs.next()){

            	if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Facts handled: " + (i-1));}

                int occurrenceId = rs.getInt("OccurrenceId");

                int newTaxonId = rs.getInt("taxonId");

                String notes = nullSafeTrim(rs.getString("occNotes"));

                Integer emStatusId = nullSafeInt(rs, "emOccurSumCatId");