EDIT

/**

 * Copyright (C) 2017 EDIT

 * European Distributed Institute of Taxonomy

 * http://www.e-taxonomy.eu

 *

 * The contents of this file are subject to the Mozilla Public License Version 1.1

 * See LICENSE.TXT at the top of this package for the full license terms.

 */

package eu.etaxonomy.taxeditor.editor.descriptiveDataSet.matrix;

import java.io.File;

import java.util.ArrayList;

import java.util.Collection;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Properties;

import java.util.Set;

import java.util.TreeSet;

import java.util.UUID;

import java.util.stream.Collectors;

import javax.inject.Inject;

import org.apache.commons.collections4.map.LinkedMap;

import org.eclipse.core.runtime.ICoreRunnable;

import org.eclipse.core.runtime.IProgressMonitor;

import org.eclipse.core.runtime.SubMonitor;

import org.eclipse.core.runtime.jobs.IJobChangeEvent;

import org.eclipse.core.runtime.jobs.Job;

import org.eclipse.core.runtime.jobs.JobChangeAdapter;

import org.eclipse.e4.core.di.annotations.Optional;

import org.eclipse.e4.ui.di.UIEventTopic;

import org.eclipse.e4.ui.di.UISynchronize;

import org.eclipse.e4.ui.services.EMenuService;

import org.eclipse.jface.layout.GridDataFactory;

import org.eclipse.jface.viewers.IStructuredSelection;

import org.eclipse.jface.viewers.StructuredSelection;

import org.eclipse.nebula.widgets.nattable.NatTable;

import org.eclipse.nebula.widgets.nattable.config.AbstractUiBindingConfiguration;

import org.eclipse.nebula.widgets.nattable.config.ConfigRegistry;

import org.eclipse.nebula.widgets.nattable.config.DefaultNatTableStyleConfiguration;

import org.eclipse.nebula.widgets.nattable.coordinate.PositionCoordinate;

import org.eclipse.nebula.widgets.nattable.coordinate.Range;

import org.eclipse.nebula.widgets.nattable.copy.command.InternalCopyDataCommandHandler;

import org.eclipse.nebula.widgets.nattable.copy.command.InternalPasteDataCommandHandler;

import org.eclipse.nebula.widgets.nattable.data.IDataProvider;

import org.eclipse.nebula.widgets.nattable.data.IRowDataProvider;

import org.eclipse.nebula.widgets.nattable.data.ListDataProvider;

import org.eclipse.nebula.widgets.nattable.export.command.ExportCommandHandler;

import org.eclipse.nebula.widgets.nattable.extension.e4.selection.E4SelectionListener;

import org.eclipse.nebula.widgets.nattable.extension.glazedlists.GlazedListsEventLayer;

import org.eclipse.nebula.widgets.nattable.extension.glazedlists.GlazedListsSortModel;

import org.eclipse.nebula.widgets.nattable.extension.glazedlists.tree.GlazedListTreeData;

import org.eclipse.nebula.widgets.nattable.extension.glazedlists.tree.GlazedListTreeRowModel;

import org.eclipse.nebula.widgets.nattable.freeze.CompositeFreezeLayer;

import org.eclipse.nebula.widgets.nattable.freeze.FreezeHelper;

import org.eclipse.nebula.widgets.nattable.freeze.FreezeLayer;

import org.eclipse.nebula.widgets.nattable.grid.GridRegion;

import org.eclipse.nebula.widgets.nattable.grid.command.ClientAreaResizeCommand;

import org.eclipse.nebula.widgets.nattable.grid.data.DefaultColumnHeaderDataProvider;

import org.eclipse.nebula.widgets.nattable.grid.data.DefaultCornerDataProvider;

import org.eclipse.nebula.widgets.nattable.grid.data.DefaultRowHeaderDataProvider;

import org.eclipse.nebula.widgets.nattable.grid.layer.ColumnHeaderLayer;

import org.eclipse.nebula.widgets.nattable.grid.layer.CornerLayer;

import org.eclipse.nebula.widgets.nattable.grid.layer.DefaultRowHeaderDataLayer;

import org.eclipse.nebula.widgets.nattable.grid.layer.GridLayer;

import org.eclipse.nebula.widgets.nattable.grid.layer.RowHeaderLayer;

import org.eclipse.nebula.widgets.nattable.group.ColumnGroupGroupHeaderLayer;

import org.eclipse.nebula.widgets.nattable.group.ColumnGroupHeaderLayer;

import org.eclipse.nebula.widgets.nattable.group.ColumnGroupModel;

import org.eclipse.nebula.widgets.nattable.layer.AbstractLayer;

import org.eclipse.nebula.widgets.nattable.layer.DataLayer;

import org.eclipse.nebula.widgets.nattable.layer.ILayer;

import org.eclipse.nebula.widgets.nattable.layer.cell.ILayerCell;

import org.eclipse.nebula.widgets.nattable.persistence.gui.PersistenceDialog;

import org.eclipse.nebula.widgets.nattable.selection.SelectionLayer;

import org.eclipse.nebula.widgets.nattable.selection.command.SelectRowsCommand;

import org.eclipse.nebula.widgets.nattable.sort.SortHeaderLayer;

import org.eclipse.nebula.widgets.nattable.sort.config.SingleClickSortConfiguration;

import org.eclipse.nebula.widgets.nattable.style.theme.ModernNatTableThemeConfiguration;

import org.eclipse.nebula.widgets.nattable.tooltip.NatTableContentTooltip;

import org.eclipse.nebula.widgets.nattable.tree.ITreeRowModel;

import org.eclipse.nebula.widgets.nattable.tree.TreeLayer;

import org.eclipse.nebula.widgets.nattable.tree.command.TreeExpandToLevelCommand;

import org.eclipse.nebula.widgets.nattable.ui.binding.UiBindingRegistry;

import org.eclipse.nebula.widgets.nattable.ui.matcher.MouseEventMatcher;

import org.eclipse.nebula.widgets.nattable.ui.menu.PopupMenuAction;

import org.eclipse.nebula.widgets.nattable.ui.menu.PopupMenuBuilder;

import org.eclipse.nebula.widgets.nattable.util.GUIHelper;

import org.eclipse.swt.SWT;

import org.eclipse.swt.layout.GridData;

import org.eclipse.swt.layout.GridLayout;

import org.eclipse.swt.widgets.Button;

import org.eclipse.swt.widgets.Composite;

import org.eclipse.swt.widgets.Event;

import org.eclipse.swt.widgets.Menu;

import ca.odell.glazedlists.BasicEventList;

import ca.odell.glazedlists.EventList;

import ca.odell.glazedlists.SortedList;

import ca.odell.glazedlists.TreeList;

import eu.etaxonomy.cdm.api.application.CdmApplicationState;

import eu.etaxonomy.cdm.api.service.IDescriptiveDataSetService;

import eu.etaxonomy.cdm.api.service.IOccurrenceService;

import eu.etaxonomy.cdm.api.service.UpdateResult;

import eu.etaxonomy.cdm.api.service.config.RemoveDescriptionsFromDescriptiveDataSetConfigurator;

import eu.etaxonomy.cdm.api.service.dto.FieldUnitDTO;

import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;

import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;

import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;

import eu.etaxonomy.cdm.common.monitor.IRemotingProgressMonitor;

import eu.etaxonomy.cdm.model.common.CdmBase;

import eu.etaxonomy.cdm.model.description.DescriptionBase;

import eu.etaxonomy.cdm.model.description.DescriptiveDataSet;

import eu.etaxonomy.cdm.model.description.SpecimenDescription;

import eu.etaxonomy.cdm.model.occurrence.FieldUnit;

import eu.etaxonomy.cdm.persistence.dto.DescriptiveDataSetBaseDto;

import eu.etaxonomy.cdm.persistence.dto.FeatureDto;

import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;

import eu.etaxonomy.cdm.persistence.dto.TaxonNodeDto;

import eu.etaxonomy.cdm.persistence.dto.TermDto;

import eu.etaxonomy.cdm.persistence.dto.TermNodeDto;

import eu.etaxonomy.cdm.persistence.dto.TermTreeDto;

import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;

import eu.etaxonomy.taxeditor.editor.internal.TaxeditorEditorPlugin;

import eu.etaxonomy.taxeditor.editor.l10n.Messages;

import eu.etaxonomy.taxeditor.event.WorkbenchEventConstants;

import eu.etaxonomy.taxeditor.model.MessagingUtils;

import eu.etaxonomy.taxeditor.operation.IFeedbackGenerator;

import eu.etaxonomy.taxeditor.session.ICdmEntitySession;

import eu.etaxonomy.taxeditor.store.CdmStore;

import eu.etaxonomy.taxeditor.workbench.WorkbenchUtility;

/**

 * Character matrix editor for editing specimen/taxon descriptions in a table

 * @author pplitzner

 * @since Nov 26, 2017

 */

public class CharacterMatrix extends Composite {

    private static final String CHARACTER_MATRIX_STATE_PROPERTIES = "characterMatrixState.properties"; //$NON-NLS-1$

    static final int LEADING_COLUMN_COUNT = 4;

    static final String TAXON_COLUMN = "taxon_column"; //$NON-NLS-1$

    static final String COLLECTOR_COLUMN = "collector_column"; //$NON-NLS-1$

    static final String IDENTIFIER_COLUMN = "identifier_column"; //$NON-NLS-1$

    static final String COUNTRY_COLUMN = "country_column"; //$NON-NLS-1$

    static final String LABEL_TAXON_ROW = "TAXON_ROW"; //$NON-NLS-1$

    static final String LABEL_TAXON_AGGREGATED_DESCRIPTION = "TAXON_AGGREGATED_DESCRIPTION"; //$NON-NLS-1$

    static final String LABEL_TAXON_AGGREGATED_DESCRIPTION_ICON = "TAXON_AGGREGATED_DESCRIPTION_ICON"; //$NON-NLS-1$

    static final String LABEL_TAXON_DEFAULT_DESCRIPTION = "TAXON_DEFAULT_DESCRIPTION"; //$NON-NLS-1$

    static final String LABEL_TAXON_DEFAULT_DESCRIPTION_ICON = "TAXON_DEFAULT_DESCRIPTION_ICON"; //$NON-NLS-1$

    static final String LABEL_TAXON_LITERATURE_DESCRIPTION = "TAXON_LITERATURE_DESCRIPTION"; //$NON-NLS-1$

    static final String LABEL_TAXON_LITERATURE_DESCRIPTION_ICON = "TAXON_LITERATURE_DESCRIPTION_ICON"; //$NON-NLS-1$

    static final String LABEL_TAXON_DESCRIPTION = "LABEL_TAXON_DESCRIPTION"; //$NON-NLS-1$

    static final String LABEL_DESCRIPTION_HAS_SUPPLEMENTAL_DATA = "LABEL_DESCRIPTION_HAS_SUPPLEMENTAL_DATA"; //$NON-NLS-1$

    boolean isFirstCall = true;

    @Inject

    private UISynchronize sync;

    @Inject

    private EMenuService menuService;

    private NatTable natTable;

    private HashMap<UUID, RowWrapperDTO<?>> rowsToMerge = new HashMap<>();

    private Map<Integer, FeatureDto> indexToFeatureMap = new HashMap<>();

    private Map<UUID, Integer> featureToIndexMap = new HashMap<>();

    private Map<UUID, List<TermDto>> categoricalFeatureToStateMap = new HashMap<>();

    private LinkedMap<String, String> propertyToLabelMap = new LinkedMap<>();

    private EventList<Object> descriptions;

    private Collection<SpecimenNodeWrapper> specimenCache = null;

    private Map<FeatureDto, CategoricalDataHistogram> featureToHistogramMap = new HashMap<>();

    private ListDataProvider<Object> bodyDataProvider;

    private FreezeLayer freezeLayer;

    private List<FeatureDto> features;

    private CharacterMatrixPart part;

    private AbstractLayer topMostLayer;

    private ConfigRegistry configRegistry;

    private MatrixBodyLayerStack bodyLayer;

    private SelectionLayer selectionLayer;

    private boolean isTreeView = true;

    private CharacterMatrixToolbar toolbar;

    private DescriptionTreeFormat treeFormat;

    private Map<RemoveDescriptionsFromDescriptiveDataSetConfigurator, List<UUID>> descriptionUuidsToDelete;

    private List<SpecimenRowWrapperDTO> specimenToAdd;

    private MouseEventMatcher mouseEventMatcher = null;

    private Set<DescriptionBase<?>> descriptionsToSave = new HashSet<>();

    public CharacterMatrix(Composite parent, CharacterMatrixPart part) {

        super(parent, SWT.NONE);

        this.part = part;

        this.setLayout(new GridLayout());

        createToolBar();

        natTable = new NatTable(this, false);

    }

    private void createToolBar(){

        toolbar = new CharacterMatrixToolbar(this, SWT.NONE);

    }

    private void applyStyles(){

        ModernNatTableThemeConfiguration configuration = new ModernNatTableThemeConfiguration();

        // NOTE: Getting the colors and fonts from the GUIHelper ensures that

        // they are disposed properly (required by SWT)

        configuration.cHeaderBgColor = GUIHelper.getColor(211, 211, 211);

        configuration.rHeaderBgColor = GUIHelper.getColor(211, 211, 211);

        natTable.addConfiguration(configuration);

    }

    void toggleTreeFlat(boolean isTree, Button btnToggleFlat, Button btnToggleTree, Button btnCollapseAll, Button btnExpandAll, Button btnFreezeSuppInfo) {

        isTreeView = isTree;

        createTable(isTree, freezeLayer.isFrozen(), true);

        btnToggleFlat.setEnabled(isTree);

        btnToggleTree.setEnabled(!isTree);

        btnCollapseAll.setEnabled(isTree);

        btnExpandAll.setEnabled(isTree);

    }

    public boolean isTreeView() {

        return isTreeView;

    }

    public void createTable(boolean treeView, boolean freezeSupplementalColumns, boolean isInitialExpandToDeepestTaxonLevel){

        /**

         * layers

         */

        createLayers(treeView);

        /**

         * configuration

         */

        configureNatTable(treeView, configRegistry, topMostLayer);

        /**

         * handlers and listeners

         */

        registerHandlersAndListeners();

        //grab all space

        GridDataFactory.fillDefaults().grab(true, true).applyTo(natTable);

        //update label to current dataset

        toolbar.getWsLabel().setText(getDescriptiveDataSet().getTitleCache());

        toolbar.getWsLabel().setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false));

        toolbar.getWsLabel().getParent().layout();

        //initial freeze of supplemental columns

        freezeSupplementalColumns(freezeSupplementalColumns);

        //add tooltip to table

//        new CategoricalChartTooltip(this);

//        new QuantitativeChartTooltip(this);

        this.layout();

        natTable.doCommand(new ClientAreaResizeCommand(natTable));

        // expand all taxa

        if(isInitialExpandToDeepestTaxonLevel){

            Integer deepestTaxonLevel = treeFormat.getDeepestTaxonLevel();

            if(deepestTaxonLevel!=null){

                natTable.doCommand(new TreeExpandToLevelCommand(deepestTaxonLevel-2));

            }

        }

        new NatTableContentTooltip(natTable, GridRegion.BODY) {

    		protected String getText(Event event) {

    			int col = this.natTable.getColumnPositionByX(event.x);

    	        int row = this.natTable.getRowPositionByY(event.y);

    	        ILayerCell cell = this.natTable.getCellByPosition(col, row);

    	        if (cell.getConfigLabels().getLabels().contains(CharacterMatrixConfigLabelAccumulator.NOT_EDITABLE)){

    	        	Object o = bodyDataProvider.getRowObject(row);

    	        	String object = "";

    	        	if (o instanceof TaxonNodeDto || o instanceof TaxonRowWrapperDTO){

    	        		object = "taxon";

    	        	}else {

    	        		object = "specimen";

    	        	}

    	        	return "Multiple data exist. Editing only possible in factual data view of " + object;

    	        }

    	        if (cell.getConfigLabels().getLabels().contains(CharacterMatrixConfigLabelAccumulator.NOT_APPLICABLE)){

    	        	return "This character is not applicable because of the state of a parent feature." ;

    	        }

    			return null;

    		}

    	};

        // clean up table state

        getNatTableState().remove(NatTable.INITIAL_PAINT_COMPLETE_FLAG);

        getNatTableState().remove(PersistenceDialog.ACTIVE_VIEW_CONFIGURATION_KEY);

    }

    private List<FeatureDto> initFeatureList(TermNodeDto node){

        List<FeatureDto> features = new ArrayList<>();

        List<TermNodeDto> childNodes = node.getChildren();

        for (TermNodeDto childNode : childNodes) {

            if (childNode != null){

                features.add((FeatureDto) childNode.getTerm());

                features.addAll(initFeatureList(childNode));

            }

        }

        return features;

    }

    public void initDescriptiveDataSet(){

        //get features/columns stored in descriptive dataset

        TermTreeDto tree = getDescriptiveDataSet().getDescriptiveSystem();

        features = initFeatureList(tree.getRoot());

        Set<FeatureDto> duplicateFeatures = features.stream().filter(i -> Collections.frequency(features, i) >1)

        .collect(Collectors.toSet());

        if (!duplicateFeatures.isEmpty()) {

            throw new IllegalArgumentException("Duplicate features found: "

                    + duplicateFeatures.stream().map(feature -> feature.getRepresentation_L10n()).collect(Collectors.joining(",")));

        }

        //init state data for categorical features

//        features.forEach(feature->fetchSupportedStates(feature));

        fetchSupportedStates(features);

        descriptions = new BasicEventList<>();

    }

    private void fetchSupportedStates(List<FeatureDto> features) {

        Set<UUID> featureUuids = new HashSet<>();

        features.forEach(i->featureUuids.add(i.getUuid()));

        categoricalFeatureToStateMap = CdmStore.getService(IDescriptiveDataSetService.class).getSupportedStatesForFeature(featureUuids);

    }

    private void createLayers(boolean treeView) {

        SortedList<Object> sortedList = new SortedList<>(descriptions, new MatrixRowComparator());

        // wrap the SortedList with the TreeList

        treeFormat = new DescriptionTreeFormat(getDescriptiveDataSet());

        @SuppressWarnings("unchecked")

        TreeList<Object> treeList = new TreeList<Object>(sortedList, treeFormat, TreeList.NODES_START_COLLAPSED);

        // wrap the SortedList with the TreeList

//        treeFormat = new DescriptionTreeFormat(getDescriptiveDataSet());

        /**

         * data provider

         */

        SpecimenColumnPropertyAccessor columnPropertyAccessor = new SpecimenColumnPropertyAccessor(this);

        bodyDataProvider = treeView?new ListDataProvider<>(treeList, columnPropertyAccessor):new ListDataProvider<>(sortedList, columnPropertyAccessor);

        DataLayer bodyDataLayer = new DataLayer(bodyDataProvider);

        bodyDataLayer.registerCommandHandler(new CopyPasteUpdateDataCommandHandler(bodyDataLayer));

        /**

         * BODY layer

         *

         *

        ViewportLayer

             ^

        TreeLayer (default visible)

             ^

        CompositeFreezeLayer

         - viewportLayer

         - selectionLayer

         - freezeLayer

             ^

        FreezeLayer

             ^

        SelectionLayer

             ^

        ColumnHideShowLayer

             ^

        DataLayer

         *

         */

        configRegistry = new ConfigRegistry();

        //register labels

        CharacterMatrixConfigLabelAccumulator labelAccumulator = new CharacterMatrixConfigLabelAccumulator(this);

        bodyDataLayer.setConfigLabelAccumulator(labelAccumulator);

        propertyToLabelMap.put(TAXON_COLUMN, Messages.CharacterMatrix_TAXON);

        propertyToLabelMap.put(COLLECTOR_COLUMN, Messages.CharacterMatrix_COLLECTOR_NO);

        propertyToLabelMap.put(IDENTIFIER_COLUMN, Messages.CharacterMatrix_IDENTIFIER);

        propertyToLabelMap.put(COUNTRY_COLUMN, Messages.CharacterMatrix_COUNTRY);

        for(int i=0;i<features.size();i++){

            FeatureDto feature = features.get(i);

            initLabels(i, feature);

        }

        // layer for event handling of GlazedLists and PropertyChanges

        GlazedListsEventLayer<Object> eventLayer = new GlazedListsEventLayer<>(bodyDataLayer, (EventList<Object>)bodyDataProvider.getList());

        GlazedListTreeData<?> treeData = new GlazedListTreeData<>(treeList);

        ITreeRowModel<?> treeRowModel = new GlazedListTreeRowModel<>(treeData);

        // assemble the column groups

        LinkedList<ColumnGroupWrapper> columnGroups = new LinkedList<>();

        List<TermNodeDto> rootChildren = getDescriptiveDataSet().getDescriptiveSystem().getRoot().getChildren();

        buildHeader(rootChildren, columnGroups);

        bodyLayer = new MatrixBodyLayerStack(eventLayer, columnGroups);

        selectionLayer = bodyLayer.getSelectionLayer();

        freezeLayer = new FreezeLayer(selectionLayer);

        final CompositeFreezeLayer compositeFreezeLayer = new CompositeFreezeLayer(

                freezeLayer, bodyLayer.getViewportLayer(), selectionLayer);

        TreeLayer treeLayer = null;

        if (treeView){

            treeLayer = new TreeLayer(compositeFreezeLayer, treeRowModel);

        }

        topMostLayer = treeView?treeLayer:compositeFreezeLayer;

        /**

         * column header layer

         */

        IDataProvider columnHeaderDataProvider = new DefaultColumnHeaderDataProvider(

                propertyToLabelMap.values().toArray(new String[] {}), propertyToLabelMap);

        DataLayer columnHeaderDataLayer = new DataLayer(columnHeaderDataProvider);

        ColumnHeaderLayer columnHeaderLayer = new ColumnHeaderLayer(columnHeaderDataLayer, topMostLayer, selectionLayer);

        ILayer topHeaderLayer = columnHeaderLayer;

        if(!columnGroups.isEmpty()){

            // first group level

            ColumnGroupWrapper groupLevel1 = columnGroups.get(0);

            ColumnGroupHeaderLayer groupLayerLevel1 = null;

            ColumnGroupGroupHeaderLayer groupLayerLevel2 = null;

            groupLayerLevel1 = new ColumnGroupHeaderLayer(columnHeaderLayer, selectionLayer, groupLevel1.getModel());

            for (Entry<TermNodeDto, TreeSet<Integer>> entry: groupLevel1.getColumnGroupToIndexMap().entrySet()) {

                TermNodeDto group = entry.getKey();

                TreeSet<Integer> indexList = entry.getValue();

                int[] intArray = indexList.stream().mapToInt(Integer::intValue).toArray();

                groupLayerLevel1.addColumnsIndexesToGroup(group.getTerm().getTitleCache(), intArray);

                groupLayerLevel1.setGroupUnbreakable(indexList.iterator().next());

            }

            topHeaderLayer = groupLayerLevel1;

            // second group level

            if(columnGroups.size()>1){

                ColumnGroupWrapper groupLevel2 = columnGroups.get(1);

                groupLayerLevel2 = new ColumnGroupGroupHeaderLayer(groupLayerLevel1, selectionLayer, groupLevel2.getModel());

                for (Entry<TermNodeDto, TreeSet<Integer>> entry: groupLevel2.getColumnGroupToIndexMap().entrySet()) {

                    TermNodeDto group = entry.getKey();

                    TreeSet<Integer> indexList = entry.getValue();

                    int[] intArray = indexList.stream().mapToInt(Integer::intValue).toArray();

                    groupLayerLevel2.addColumnsIndexesToGroup(group.getTerm().getTitleCache(), intArray);

                    groupLayerLevel2.setGroupUnbreakable(indexList.iterator().next());

                }

                topHeaderLayer = groupLayerLevel2;

            }

        }

        // add the SortHeaderLayer to the column header layer stack

        // as we use GlazedLists, we use the GlazedListsSortModel which

        // delegates the sorting to the SortedList

        final SortHeaderLayer<DescriptionBase<?>> sortHeaderLayer = new SortHeaderLayer<>(

                topHeaderLayer,

                new GlazedListsSortModel<>(

                        sortedList,

                        columnPropertyAccessor,

                        configRegistry,

                        columnHeaderDataLayer));

        /**

         * row header layer

         */

        IDataProvider rowHeaderDataProvider = new DefaultRowHeaderDataProvider(bodyDataProvider);

        DefaultRowHeaderDataLayer rowHeaderDataLayer = new DefaultRowHeaderDataLayer(rowHeaderDataProvider);

        RowHeaderLayer rowHeaderLayer = new RowHeaderLayer(rowHeaderDataLayer,

                topMostLayer, selectionLayer);

        /**

         * corner layer

         */

        ILayer cornerLayer = new CornerLayer(

                new DataLayer(new DefaultCornerDataProvider(columnHeaderDataProvider, rowHeaderDataProvider)),

                rowHeaderLayer, sortHeaderLayer);

        /**

         * GRID layer (composition of all other layers)

         */

        GridLayer gridLayer = new GridLayer(topMostLayer, sortHeaderLayer, rowHeaderLayer, cornerLayer);

        natTable.setLayer(gridLayer);

    }

    private TreeSet<Integer> recurseChildIndexes(TermNodeDto node){

        TreeSet<Integer> childIndexes = new TreeSet<>();

        if(node.getChildren().size() >0){

            List<TermNodeDto> childNodes = node.getChildren();

            for (TermNodeDto childNode: childNodes) {

                if (childNode != null){

                    childIndexes.addAll(recurseChildIndexes(childNode));

                }

            }

        }

        childIndexes.add(features.indexOf(node.getTerm())+LEADING_COLUMN_COUNT);

        return childIndexes;

    }

    private void buildHeader(List<TermNodeDto> nodes, LinkedList<ColumnGroupWrapper> columnGroups){

        Map<TermNodeDto, TreeSet<Integer>> columnGroupToIndexMap = new HashMap<>();

        List<TermNodeDto> childNodes = new ArrayList<>();

        for (TermNodeDto node : nodes) {

            if (node != null){

                TreeSet<Integer> childIndexes = recurseChildIndexes(node);

                if(childIndexes.size()>1){

                    // filter out groups that only have one member

                    columnGroupToIndexMap.put(node, childIndexes);

                }

                childNodes.addAll(node.getChildren());

            }

        }

        if(!columnGroupToIndexMap.isEmpty()){

            columnGroups.addFirst(new ColumnGroupWrapper(new ColumnGroupModel(), columnGroupToIndexMap));

        }

        if(!childNodes.isEmpty()){

            buildHeader(childNodes, columnGroups);

        }

    }

    private void registerHandlersAndListeners() {

        natTable.registerCommandHandler(new ExportCommandHandler(natTable));

        //selection listener

        E4SelectionListener<Object> selectionListener = new CellSelectionListener(part.getSelectionService(),

                bodyLayer.getSelectionLayer(), bodyDataProvider, part);

        bodyLayer.getSelectionLayer().addLayerListener(selectionListener);

        selectionListener.setFullySelectedRowsOnly(false);

        //register handler for view configuration menu

        natTable.registerCommandHandler(toolbar.getDisplayPersistenceDialogCommandHandler());

        //register handlers for copy&paste

        natTable.registerCommandHandler(

                new InternalPasteDataCommandHandler(bodyLayer.getSelectionLayer(), natTable.getInternalCellClipboard()));

        natTable.registerCommandHandler(

                new InternalCopyDataCommandHandler(bodyLayer.getSelectionLayer(), natTable.getInternalCellClipboard()));

    }

    private void configureNatTable(boolean treeView,

            ConfigRegistry configRegistry,

            AbstractLayer topMostLayer) {

        /**

         * CONFIGURATION

         */

      //+++CONTEXT MENU+++

        menuService.registerContextMenu(natTable, "eu.etaxonomy.taxeditor.editor.popupmenu.charactermatrix"); //$NON-NLS-1$

        // get the menu registered by EMenuService

        final Menu e4Menu = natTable.getMenu();

        natTable.setConfigRegistry(configRegistry);

        applyStyles();

        //add default configuration because autoconfigure is set to false in constructor

        natTable.addConfiguration(new DefaultNatTableStyleConfiguration());

        // this is for DEBUG ONLY

//                natTable.addConfiguration(new DebugMenuConfiguration(natTable));

        // override the default sort configuration and change the mouse bindings

        // to sort on a single click

        if(!treeView){

            natTable.addConfiguration(new SingleClickSortConfiguration());

        }

        natTable.addConfiguration(new CharacterMatrixLabelStyleConfiguration());

        // add the header menu configuration for adding the column header menu

        // with hide/show actions

//        natTable.addConfiguration(new CharacterMatrixHeaderMenuConfiguration(natTable));

        // add custom configuration for data conversion and add column labels to viewport layer

        topMostLayer.addConfiguration(new CellEditorDataConversionConfiguration(this));

        //copy&paste configuration

        natTable.addConfiguration(new CopyPasteEditBindings(bodyLayer.getSelectionLayer(), natTable.getInternalCellClipboard()));

        // remove the menu reference from NatTable instance

        natTable.setMenu(null);

//        natTable.addConfiguration(new CharacterMatrixHeaderMenuConfiguration(natTable));

        natTable.configure();

        natTable.addConfiguration(

                    new AbstractUiBindingConfiguration() {

                @Override

                public void configureUiBindings(

                        UiBindingRegistry uiBindingRegistry) {

                    // add e4 menu to NatTable

                    uiBindingRegistry.unregisterMouseDownBinding(mouseEventMatcher);

                    if (isFirstCall){

	                    new PopupMenuBuilder(natTable, e4Menu)

	                    .withHideColumnMenuItem()

	                    .withShowAllColumnsMenuItem()

	                    .build();

	                    isFirstCall = false;

                    }else{

                    	new PopupMenuBuilder(natTable, e4Menu)

	                    .build();

                    }

                    // register the UI binding for header, corner and body region

                    mouseEventMatcher = new MouseEventMatcher(

                            SWT.NONE,

                            null,

                            MouseEventMatcher.RIGHT_BUTTON);

                    uiBindingRegistry.registerMouseDownBinding(mouseEventMatcher,

                            new PopupMenuAction(e4Menu));

                }

            });

    }

    void freezeSupplementalColumns(boolean freeze){

        int rightMostFreezeColumIndex = LEADING_COLUMN_COUNT-1;

        Collection<Integer> hiddenColumnIndexes = bodyLayer.getColumnHideShowLayer().getHiddenColumnIndexes();

        for (Integer integer : hiddenColumnIndexes) {

            if(integer<LEADING_COLUMN_COUNT){

                rightMostFreezeColumIndex--;

            }

        }

        if(freeze){

            FreezeHelper.freeze(freezeLayer, bodyLayer.getViewportLayer(),

                    new PositionCoordinate(bodyLayer.getViewportLayer(), 0, 0),

                    new PositionCoordinate(bodyLayer.getViewportLayer(), rightMostFreezeColumIndex, -1));

        }

        else{

            FreezeHelper.unfreeze(freezeLayer, bodyLayer.getViewportLayer());

        }

    }

    private void initLabels(int index, FeatureDto feature) {

        indexToFeatureMap.put(index+LEADING_COLUMN_COUNT, feature);

        featureToIndexMap.put(feature.getUuid(), index+LEADING_COLUMN_COUNT);

        String label = feature.getTitleCache();

        String property = feature.getUuid().toString();

        //show unit for quantitative data

        if(feature.isSupportsQuantitativeData()){

            Set<TermDto> recommendedMeasurementUnits = feature.getRecommendedMeasurementUnits();

//            if(recommendedMeasurementUnits.size()>1){

//                MessagingUtils.warningDialog(Messages.CharacterMatrix_INIT_PROBLEM, CharacterMatrix.class,

//                        String.format(Messages.CharacterMatrix_INIT_PROBLEM_MESSAGE, feature.getLabel()));

//            }

            if(recommendedMeasurementUnits.size()==1){

                TermDto unit = recommendedMeasurementUnits.iterator().next();

                label += " ["+unit.getIdInVocabulary()+"]"; //$NON-NLS-1$ //$NON-NLS-2$

            }

        }

        propertyToLabelMap.put(property, label);

    }

    public void loadDescriptions(boolean isInitialExpandToDeepestTaxonLevel, boolean initialLoading) {

        UUID monitorUuid =  CdmApplicationState.getLongRunningTasksService().monitGetRowWrapper(this.getDescriptiveDataSet().getUuid());

        final Collection<RowWrapperDTO<?>> wrappers = new ArrayList<>();

        String jobLabel = Messages.CharacterMatrix_LOAD_CHARACTER_DATA;

        Job job = Job.create(jobLabel, (ICoreRunnable) monitor -> {

            SubMonitor subMonitor = SubMonitor.convert(monitor);

            subMonitor.beginTask(jobLabel, IProgressMonitor.UNKNOWN);

            IRemotingProgressMonitor remotingMonitor;

            try {

                 remotingMonitor = CdmStore.getProgressMonitorClientManager()

                .pollMonitor(jobLabel,

                        monitorUuid,

                        50,

                        null,

                        (List<IFeedbackGenerator>)null,

                        subMonitor);

            } catch (InterruptedException e) {

                MessagingUtils.informationDialog(Messages.CharacterMatrix_LOADING_FAILED_TITLE,

                        Messages.CharacterMatrix_LOADING_FAILED_MESSAGE);

                return;

            }

            Object result = remotingMonitor.getResult();

            if(result instanceof Collection){

                wrappers.addAll((Collection<RowWrapperDTO<?>>) result);

            }

            if(result instanceof Exception){

                MessagingUtils.errorDialog("Exception during description loading", this.getClass(), "An exception occured during loading", TaxeditorEditorPlugin.PLUGIN_ID, (Throwable) result, true);

            }

            else if(wrappers.isEmpty()){

                MessagingUtils.informationDialog(Messages.CharacterMatrix_NO_DESCRIPTION_TITLE,

                        Messages.CharacterMatrix_NO_DESCRIPTION_MESSAGE);

            }

            monitor.done();

        });

        job.addJobChangeListener(new JobChangeAdapter(){

            @Override

            public void done(IJobChangeEvent event) {

                sync.syncExec(()->{

                    List<RowWrapperDTO> rowsWithoutTaxonNode = wrappers.stream().filter(row->row.getTaxonNode()==null).collect(Collectors.toList());

                    if(!rowsWithoutTaxonNode.isEmpty()){

                        String collect = rowsWithoutTaxonNode.stream().

                        map(row->row.getDescription().toString())

                        .collect(Collectors.joining("\n\n - ")); //$NON-NLS-1$

                        MessagingUtils.warningDialog(

                                Messages.CharacterMatrix_NO_NODE_FOUND_TITLE,

                                this.getClass(),

                                String.format(Messages.CharacterMatrix_NO_NODE_FOUND_MESSAGE, collect)

                                );

                    }

                    descriptions.clear();

//                    part.setDescriptiveDataSet(CdmStore.getService(IDescriptiveDataSetService.class).getDescriptiveDataSetDtoByUuid(part.getDescriptiveDataSet().getUuid()));

                    wrappers.stream().filter(row->row.getTaxonNode()!=null).forEach(wrapper->CharacterMatrix.this.descriptions.add(wrapper));

                    if(initialLoading){

                        loadingDone(isInitialExpandToDeepestTaxonLevel);

                    }

                });

            }

        });

        job.schedule();

    }

    public IStructuredSelection getSelection(){

        Set<Range> selectedRowPositions = bodyLayer.getSelectionLayer().getSelectedRowPositions();

        List<Object> selectedObjects = new ArrayList<>();

        for (Range range : selectedRowPositions) {

            for(int i=range.start;i<range.end;i++){

                selectedObjects.add(bodyDataProvider.getRowObject(i));

            }

        }

        return new StructuredSelection(selectedObjects);

    }

    private void loadingDone(boolean isInitialExpandToDeepestTaxonLevel) {

        this.part.loadingDone();

        createTable(isTreeView, freezeLayer.isFrozen(), isInitialExpandToDeepestTaxonLevel);

    }

    public List<TermDto> getSupportedStatesForCategoricalFeature(UUID featureUuid){

        return categoricalFeatureToStateMap.get(featureUuid);

    }

    public Set<DescriptionBase<?>> getDescriptionsToSave() {

        return descriptionsToSave;

    }

    public void addDescriptionToSave(DescriptionBase<?> descriptionToSave) {

        this.descriptionsToSave.add(descriptionToSave);

    }

    public Map<Integer, FeatureDto> getIndexToFeatureMap() {

        return indexToFeatureMap;

    }

    public Map<UUID, Integer> getFeatureToIndexMap() {

		return featureToIndexMap;

	}

	public void setFeatureToIndexMap(Map<UUID, Integer> featureToIndexMap) {

		this.featureToIndexMap = featureToIndexMap;

	}

	public LinkedMap<String, String> getPropertyToLabelMap() {

        return propertyToLabelMap;

    }

    public void setDirty() {

        part.setDirty();

    }

    public Map<RemoveDescriptionsFromDescriptiveDataSetConfigurator, List<UUID>> getDescriptionsToDelete() {

        return descriptionUuidsToDelete;

    }

    public void addDescriptionToDelete(UUID descriptionToDelete, RemoveDescriptionsFromDescriptiveDataSetConfigurator config) {

        if (descriptionUuidsToDelete == null){

            descriptionUuidsToDelete = new HashMap<>();

        }

        if (descriptionUuidsToDelete.get(config) != null){

            descriptionUuidsToDelete.get(config).add(descriptionToDelete);

        }else{

            List<UUID> uuidList = new ArrayList<>();

            uuidList.add(descriptionToDelete);

            this.descriptionUuidsToDelete.put(config, uuidList);

        }

    }