Merge branch 'pr-32548' into feat/datasets-folders

superset-frontend/package-lock.json

@@ -48823,6 +48823,7 @@
         "@types/react": "*",
         "@types/react-loadable": "*",
         "@types/tinycolor2": "*",
+        "nanoid": "^5.0.9",
         "react": "^17.0.2",
         "react-loadable": "^5.5.0",
         "tinycolor2": "*"

superset-frontend/packages/superset-ui-chart-controls/src/components/MetricOption.tsx

@@ -42,7 +42,7 @@ const FlexRowContainer = styled.div`
 `;
 
 export interface MetricOptionProps {
-  metric: Omit<Metric, 'id'> & { label?: string };
+  metric: Omit<Metric, 'id' | 'uuid'> & { label?: string };
   openInNewWindow?: boolean;
   showFormula?: boolean;
   showType?: boolean;

superset-frontend/packages/superset-ui-chart-controls/src/components/labelUtils.tsx

@@ -97,7 +97,7 @@ export const getColumnTooltipNode = (
   );
 };
 
-type MetricType = Omit<Metric, 'id'> & { label?: string };
+type MetricType = Omit<Metric, 'id' | 'uuid'> & { label?: string };
 
 export const getMetricTooltipNode = (
   metric: MetricType,

superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts

@@ -121,6 +121,7 @@ export const TestDataset: Dataset = {
   main_dttm_col: 'ds',
   metrics: [
     {
+      uuid: '123',
       certification_details: null,
       certified_by: null,
       d3format: null,

superset-frontend/packages/superset-ui-chart-controls/test/utils/defineSavedMetrics.test.tsx

@@ -32,6 +32,7 @@ describe('defineSavedMetrics', () => {
         {
           metric_name: 'COUNT(*) non-default-dataset-metric',
           expression: 'COUNT(*) non-default-dataset-metric',
+          uuid: '1',
         },
       ],
       type: DatasourceType.Table,
@@ -48,6 +49,7 @@ describe('defineSavedMetrics', () => {
       {
         metric_name: 'COUNT(*) non-default-dataset-metric',
         expression: 'COUNT(*) non-default-dataset-metric',
+        uuid: '1',
       },
     ]);
     // @ts-ignore

superset-frontend/packages/superset-ui-chart-controls/test/utils/mainMetric.test.ts

@@ -24,15 +24,24 @@ describe('mainMetric', () => {
     expect(mainMetric(null)).toBeUndefined();
   });
   it('prefers the "count" metric when first', () => {
-    const metrics = [{ metric_name: 'count' }, { metric_name: 'foo' }];
+    const metrics = [
+      { metric_name: 'count', uuid: '1' },
+      { metric_name: 'foo', uuid: '2' },
+    ];
     expect(mainMetric(metrics)).toBe('count');
   });
   it('prefers the "count" metric when not first', () => {
-    const metrics = [{ metric_name: 'foo' }, { metric_name: 'count' }];
+    const metrics = [
+      { metric_name: 'foo', uuid: '1' },
+      { metric_name: 'count', uuid: '2' },
+    ];
     expect(mainMetric(metrics)).toBe('count');
   });
   it('selects the first metric when "count" is not an option', () => {
-    const metrics = [{ metric_name: 'foo' }, { metric_name: 'not_count' }];
+    const metrics = [
+      { metric_name: 'foo', uuid: '2' },
+      { metric_name: 'not_count', uuid: '2' },
+    ];
     expect(mainMetric(metrics)).toBe('foo');
   });
 });

superset-frontend/packages/superset-ui-core/package.json

@@ -81,6 +81,7 @@
     "@types/react": "*",
     "@types/react-loadable": "*",
     "@types/tinycolor2": "*",
+    "nanoid": "^5.0.9",
     "react": "^17.0.2",
     "react-loadable": "^5.5.0",
     "tinycolor2": "*"

superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import { nanoid } from 'nanoid';
 import { Column } from './Column';
 import { Metric } from './Metric';
 
@@ -58,6 +59,7 @@ export const DEFAULT_METRICS: Metric[] = [
   {
     metric_name: 'COUNT(*)',
     expression: 'COUNT(*)',
+    uuid: nanoid(),
   },
 ];
 

superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts

@@ -60,6 +60,7 @@ export type SavedMetric = string;
  */
 export interface Metric {
   id?: number;
+  uuid: string;
   metric_name: string;
   expression?: Maybe<string>;
   certification_details?: Maybe<string>;

superset-frontend/packages/superset-ui-core/test/query/types/Datasource.test.ts

@@ -20,10 +20,10 @@ import { DatasourceType, DEFAULT_METRICS } from '@superset-ui/core';
 
 test('DEFAULT_METRICS', () => {
   expect(DEFAULT_METRICS).toEqual([
-    {
+    expect.objectContaining({
       metric_name: 'COUNT(*)',
       expression: 'COUNT(*)',
-    },
+    }),
   ]);
 });
 

superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts

@@ -149,6 +149,7 @@ describe('BigNumberWithTrendline', () => {
               label: 'value',
               metric_name: 'value',
               d3format: '.2f',
+              uuid: '1',
             },
           ],
         },
@@ -174,6 +175,7 @@ describe('BigNumberWithTrendline', () => {
               metric_name: 'value',
               d3format: '.2f',
               currency: { symbol: 'USD', symbolPosition: 'prefix' },
+              uuid: '1',
             },
           ],
         },

superset-frontend/src/explore/components/DatasourcePanel/DatasourceItems.tsx

@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { VariableSizeList as List } from 'react-window';
+import { cloneDeep } from 'lodash';
+import { FlattenedItem, Folder } from './types';
+import DatasourcePanelItem from './DatasourcePanelItem';
+
+const BORDER_WIDTH = 2;
+const HEADER_ITEM_HEIGHT = 50;
+const METRIC_OR_COLUMN_ITEM_HEIGHT = 32;
+const DIVIDER_ITEM_HEIGHT = 16;
+
+const flattenFolderStructure = (
+  folders: Folder[],
+  depth = 0,
+  folderMap: Map<string, Folder> = new Map(),
+): { flattenedItems: FlattenedItem[]; folderMap: Map<string, Folder> } => {
+  const flattenedItems: FlattenedItem[] = [];
+
+  folders.forEach((folder, idx) => {
+    folderMap.set(folder.id, folder);
+
+    flattenedItems.push({
+      type: 'header',
+      folderId: folder.id,
+      depth,
+      height: HEADER_ITEM_HEIGHT,
+    });
+
+    if (!folder.isCollapsed) {
+      folder.items.forEach(item => {
+        flattenedItems.push({
+          type: 'item',
+          folderId: folder.id,
+          depth,
+          item,
+          height: METRIC_OR_COLUMN_ITEM_HEIGHT,
+        });
+      });
+
+      if (folder.subFolders && folder.subFolders.length > 0) {
+        const { flattenedItems: subItems } = flattenFolderStructure(
+          folder.subFolders,
+          depth + 1,
+          folderMap,
+        );
+
+        flattenedItems.push(...subItems);
+      }
+    }
+    if (depth === 0 && idx !== folders.length - 1) {
+      flattenedItems.push({
+        type: 'divider',
+        folderId: folder.id,
+        depth,
+        height: DIVIDER_ITEM_HEIGHT,
+      });
+    }
+  });
+
+  return { flattenedItems, folderMap };
+};
+
+interface DatasourceItemsProps {
+  width: number;
+  height: number;
+  folders: Folder[];
+}
+export const DatasourceItems = ({
+  width,
+  height,
+  folders,
+}: DatasourceItemsProps) => {
+  const [folderStructure, setFolderStructure] = useState<Folder[]>(folders);
+
+  useEffect(() => {
+    setFolderStructure(prev => (prev !== folders ? folders : prev));
+  }, [folders]);
+
+  const { flattenedItems, folderMap } = useMemo(
+    () => flattenFolderStructure(folderStructure),
+    [folderStructure],
+  );
+
+  const handleToggleCollapse = useCallback((folderId: string) => {
+    setFolderStructure(prevFolders => {
+      const updatedFolders = cloneDeep(prevFolders);
+
+      const updateFolder = (folders: Folder[] | undefined): boolean => {
+        if (!folders) {
+          return false;
+        }
+        for (let i = 0; i < folders.length; i += 1) {
+          if (folders[i].id === folderId) {
+            // eslint-disable-next-line no-param-reassign
+            folders[i].isCollapsed = !folders[i].isCollapsed;
+            return true;
+          }
+
+          if (folders[i].subFolders && updateFolder(folders[i].subFolders)) {
+            return true;
+          }
+        }
+        return false;
+      };
+
+      updateFolder(updatedFolders);
+      return updatedFolders;
+    });
+  }, []);
+
+  const getItemSize = useCallback(
+    (index: number) => flattenedItems[index].height,
+    [flattenedItems],
+  );
+
+  return (
+    <List
+      width={width - BORDER_WIDTH}
+      height={height}
+      itemSize={getItemSize}
+      itemCount={flattenedItems.length}
+      itemData={{
+        flattenedItems,
+        folderMap,
+        width,
+        onToggleCollapse: handleToggleCollapse,
+      }}
+      overscanCount={5}
+    >
+      {DatasourcePanelItem}
+    </List>
+  );
+};

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanel.test.tsx

@@ -59,6 +59,48 @@ const datasource: IDatasource = {
   datasource_name: 'table1',
 };
 
+const datasourceWithFolders: IDatasource = {
+  ...datasource,
+  folders: [
+    {
+      name: 'Test folder',
+      type: 'folder',
+      uuid: '1',
+      children: [
+        {
+          name: 'Test nested folder',
+          type: 'folder',
+          uuid: '1.1',
+          children: [
+            {
+              type: 'metric',
+              uuid: metrics[0].uuid,
+              name: metrics[0].metric_name,
+            },
+          ],
+        },
+      ],
+    },
+    {
+      name: 'Second test folder',
+      type: 'folder',
+      uuid: '2',
+      children: [
+        {
+          type: 'column',
+          uuid: columns[0].uuid,
+          name: columns[0].column_name,
+        },
+        {
+          type: 'column',
+          uuid: columns[1].uuid,
+          name: columns[1].column_name,
+        },
+      ],
+    },
+  ],
+};
+
 const mockUser = {
   createdOn: '2021-04-27T18:12:38.952304',
   email: 'admin',
@@ -90,6 +132,18 @@ const props: DatasourcePanelProps = {
   width: 300,
 };
 
+const propsWithFolders = {
+  ...props,
+  datasource: datasourceWithFolders,
+  controls: {
+    ...props.controls,
+    datasource: {
+      ...props.controls.datasource,
+      datasource: datasourceWithFolders,
+    },
+  },
+};
+
 const metricProps = {
   savedMetrics: [],
   columns: [],
@@ -125,13 +179,9 @@ test('should render the metrics', async () => {
     </ExploreContainer>,
     { useRedux: true, useDnd: true },
   );
-  const metricsNum = metrics.length;
   metrics.forEach(metric =>
     expect(screen.getByText(metric.metric_name)).toBeInTheDocument(),
   );
-  expect(
-    await screen.findByText(`Showing ${metricsNum} of ${metricsNum}`),
-  ).toBeInTheDocument();
 });
 
 test('should render the columns', async () => {
@@ -142,13 +192,9 @@ test('should render the columns', async () => {
     </ExploreContainer>,
     { useRedux: true, useDnd: true },
   );
-  const columnsNum = columns.length;
   columns.forEach(col =>
     expect(screen.getByText(col.column_name)).toBeInTheDocument(),
   );
-  expect(
-    await screen.findByText(`Showing ${columnsNum} of ${columnsNum}`),
-  ).toBeInTheDocument();
 });
 
 describe('DatasourcePanel', () => {
@@ -310,3 +356,139 @@ test('should render only droppable metrics and columns', async () => {
 
   unmount();
 });
+
+test('Renders with custom folders', () => {
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  columns.forEach(col => {
+    expect(screen.getByText(col.column_name)).toBeInTheDocument();
+  });
+
+  metrics.forEach(metric => {
+    expect(screen.getByText(metric.metric_name)).toBeInTheDocument();
+  });
+
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
+  expect(screen.getAllByTestId('datasource-panel-divider').length).toEqual(3);
+});
+
+test('Collapse folders', () => {
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  userEvent.click(screen.getAllByRole('button')[0]);
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.queryByText('Test nested folder')).not.toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  expect(screen.queryByText(metrics[0].metric_name)).not.toBeInTheDocument();
+
+  userEvent.click(screen.getAllByRole('button')[0]);
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  expect(screen.getByText(metrics[0].metric_name)).toBeInTheDocument();
+});
+
+test('Default Metrics and Columns folders dont render when all metrics and columns are assigned to custom folders', () => {
+  const datasourceWithFullFolders: IDatasource = {
+    ...datasource,
+    folders: [
+      {
+        name: 'Test folder',
+        type: 'folder',
+        uuid: '1',
+        children: [
+          {
+            name: 'Test nested folder',
+            type: 'folder',
+            uuid: '1.1',
+            children: metrics.map(m => ({
+              type: 'metric' as const,
+              uuid: m.uuid,
+              name: m.metric_name,
+            })),
+          },
+        ],
+      },
+      {
+        name: 'Second test folder',
+        type: 'folder',
+        uuid: '2',
+        children: columns.map(c => ({
+          type: 'column',
+          uuid: c.uuid,
+          name: c.column_name,
+        })),
+      },
+    ],
+  };
+  const propsWithFullFolders = {
+    ...props,
+    datasource: datasourceWithFullFolders,
+    controls: {
+      ...props.controls,
+      datasource: {
+        ...props.controls.datasource,
+        datasource: datasourceWithFullFolders,
+      },
+    },
+  };
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFullFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.queryByText('Metrics')).not.toBeInTheDocument();
+  expect(screen.queryByText('Columns')).not.toBeInTheDocument();
+
+  columns.forEach(col => {
+    expect(screen.getByText(col.column_name)).toBeInTheDocument();
+  });
+
+  metrics.forEach(metric => {
+    expect(screen.getByText(metric.metric_name)).toBeInTheDocument();
+  });
+
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
+  expect(screen.getAllByTestId('datasource-panel-divider').length).toEqual(1);
+});

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelDragOption/DatasourcePanelDragOption.test.tsx

@@ -23,7 +23,7 @@ import DatasourcePanelDragOption from '.';
 test('should render', async () => {
   render(
     <DatasourcePanelDragOption
-      value={{ metric_name: 'test' }}
+      value={{ metric_name: 'test', uuid: '1' }}
       type={DndItemType.Metric}
     />,
     { useDnd: true },
@@ -38,7 +38,7 @@ test('should render', async () => {
 test('should have attribute draggable:true', async () => {
   render(
     <DatasourcePanelDragOption
-      value={{ metric_name: 'test' }}
+      value={{ metric_name: 'test', uuid: '1' }}
       type={DndItemType.Metric}
     />,
     { useDnd: true },

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelItem.test.tsx

@@ -20,178 +20,84 @@ import {
   columns,
   metrics,
 } from 'src/explore/components/DatasourcePanel/fixtures';
-import { fireEvent, render, within } from 'spec/helpers/testing-library';
+import { screen, userEvent, render } from 'spec/helpers/testing-library';
-import DatasourcePanelItem from './DatasourcePanelItem';
+import DatasourcePanelItem, {
+  DatasourcePanelItemProps,
+} from './DatasourcePanelItem';
 
-const mockData = {
+const mockData: DatasourcePanelItemProps['data'] = {
-  metricSlice: metrics,
+  flattenedItems: [
-  columnSlice: columns,
+    { type: 'header', depth: 0, folderId: '1', height: 50 },
-  totalMetrics: Math.max(metrics.length, 10),
+    ...metrics.map((m, idx) => ({
-  totalColumns: Math.max(columns.length, 13),
+      type: 'item' as const,
+      depth: 0,
+      folderId: '1',
+      height: 32,
+      index: idx,
+      item: { ...m, type: 'metric' as const },
+    })),
+    { type: 'divider', depth: 0, folderId: '1', height: 16 },
+    { type: 'header', depth: 0, folderId: '2', height: 50 },
+    ...columns.map((m, idx) => ({
+      type: 'item' as const,
+      depth: 0,
+      folderId: '2',
+      height: 32,
+      index: idx,
+      item: { ...m, type: 'column' as const },
+    })),
+  ],
+  folderMap: new Map([
+    [
+      '1',
+      {
+        id: '1',
+        isCollapsed: false,
+        name: 'Metrics',
+        items: metrics.map(m => ({ ...m, type: 'metric' })),
+      },
+    ],
+    [
+      '2',
+      {
+        id: '2',
+        isCollapsed: false,
+        name: 'Columns',
+        items: columns.map(c => ({ ...c, type: 'column' })),
+      },
+    ],
+  ]),
   width: 300,
-  showAllMetrics: false,
+  onToggleCollapse: jest.fn(),
-  onShowAllMetricsChange: jest.fn(),
-  showAllColumns: false,
-  onShowAllColumnsChange: jest.fn(),
-  collapseMetrics: false,
-  onCollapseMetricsChange: jest.fn(),
-  collapseColumns: false,
-  onCollapseColumnsChange: jest.fn(),
-  hiddenMetricCount: 0,
-  hiddenColumnCount: 0,
 };
 
-test('renders each item accordingly', () => {
+const setup = (data: DatasourcePanelItemProps['data'] = mockData) =>
-  const { getByText, getByTestId, rerender, container } = render(
+  render(
-    <DatasourcePanelItem index={0} data={mockData} style={{}} />,
+    <>
+      {data.flattenedItems.map((_, index) => (
+        <DatasourcePanelItem index={index} data={data} style={{}} />
+      ))}
+    </>,
     { useDnd: true },
   );
 
-  expect(getByText('Metrics')).toBeInTheDocument();
+test('renders each item accordingly', () => {
-  rerender(<DatasourcePanelItem index={1} data={mockData} style={{}} />);
+  setup();
-  expect(
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
-    getByText(
+  expect(screen.getByText('metric_end_certified')).toBeInTheDocument();
-      `Showing ${mockData.metricSlice.length} of ${mockData.totalMetrics}`,
+  expect(screen.getByText('metric_end')).toBeInTheDocument();
-    ),
-  ).toBeInTheDocument();
-  mockData.metricSlice.forEach((metric, metricIndex) => {
-    rerender(
-      <DatasourcePanelItem
-        index={metricIndex + 2}
-        data={mockData}
-        style={{}}
-      />,
-    );
-    expect(getByTestId('DatasourcePanelDragOption')).toBeInTheDocument();
-    expect(
-      within(getByTestId('DatasourcePanelDragOption')).getByText(
-        metric.metric_name,
-      ),
-    ).toBeInTheDocument();
-  });
-  rerender(
-    <DatasourcePanelItem
-      index={2 + mockData.metricSlice.length}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  expect(container).toHaveTextContent('');
 
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
+  expect(screen.getByText('Columns')).toBeInTheDocument();
-  rerender(
+  expect(screen.getByText('bootcamp_attend')).toBeInTheDocument();
-    <DatasourcePanelItem
+  expect(screen.getByText('calc_first_time_dev')).toBeInTheDocument();
-      index={startIndexOfColumnSection}
+  expect(screen.getByText('aaaaaaaaaaa')).toBeInTheDocument();
-      data={mockData}
+
-      style={{}}
+  expect(screen.getByTestId('datasource-panel-divider')).toBeInTheDocument();
-    />,
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
-  );
-  expect(getByText('Columns')).toBeInTheDocument();
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection + 1}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  expect(
-    getByText(
-      `Showing ${mockData.columnSlice.length} of ${mockData.totalColumns}`,
-    ),
-  ).toBeInTheDocument();
-  mockData.columnSlice.forEach((column, columnIndex) => {
-    rerender(
-      <DatasourcePanelItem
-        index={startIndexOfColumnSection + columnIndex + 2}
-        data={mockData}
-        style={{}}
-      />,
-    );
-    expect(getByTestId('DatasourcePanelDragOption')).toBeInTheDocument();
-    expect(
-      within(getByTestId('DatasourcePanelDragOption')).getByText(
-        column.column_name,
-      ),
-    ).toBeInTheDocument();
-  });
 });
 
 test('can collapse metrics and columns', () => {
-  mockData.onCollapseMetricsChange.mockClear();
+  setup();
-  mockData.onCollapseColumnsChange.mockClear();
+  userEvent.click(screen.getAllByRole('button')[0]);
-  const { queryByText, getByRole, rerender } = render(
+  expect(mockData.onToggleCollapse).toHaveBeenCalled();
-    <DatasourcePanelItem index={0} data={mockData} style={{}} />,
-    { useDnd: true },
-  );
-  fireEvent.click(getByRole('button'));
-  expect(mockData.onCollapseMetricsChange).toHaveBeenCalled();
-  expect(mockData.onCollapseColumnsChange).not.toHaveBeenCalled();
-
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  fireEvent.click(getByRole('button'));
-  expect(mockData.onCollapseColumnsChange).toHaveBeenCalled();
-
-  rerender(
-    <DatasourcePanelItem
-      index={1}
-      data={{
-        ...mockData,
-        collapseMetrics: true,
-      }}
-      style={{}}
-    />,
-  );
-  expect(
-    queryByText(
-      `Showing ${mockData.metricSlice.length} of ${mockData.totalMetrics}`,
-    ),
-  ).not.toBeInTheDocument();
-
-  rerender(
-    <DatasourcePanelItem
-      index={2}
-      data={{
-        ...mockData,
-        collapseMetrics: true,
-      }}
-      style={{}}
-    />,
-  );
-  expect(queryByText('Columns')).toBeInTheDocument();
-});
-
-test('shows ineligible items count', () => {
-  const hiddenColumnCount = 3;
-  const hiddenMetricCount = 1;
-  const dataWithHiddenItems = {
-    ...mockData,
-    hiddenColumnCount,
-    hiddenMetricCount,
-  };
-  const { getByText, rerender } = render(
-    <DatasourcePanelItem index={1} data={dataWithHiddenItems} style={{}} />,
-    { useDnd: true },
-  );
-  expect(
-    getByText(`${hiddenMetricCount} ineligible item(s) are hidden`),
-  ).toBeInTheDocument();
-
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection + 1}
-      data={dataWithHiddenItems}
-      style={{}}
-    />,
-  );
-  expect(
-    getByText(`${hiddenColumnCount} ineligible item(s) are hidden`),
-  ).toBeInTheDocument();
 });

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelItem.tsx

@@ -16,71 +16,24 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { CSSProperties, FC } from 'react';
+import { CSSProperties } from 'react';
 
-import { css, Metric, styled, t, useTheme } from '@superset-ui/core';
+import { css, styled, useTheme } from '@superset-ui/core';
 
 import Icons from 'src/components/Icons';
 import DatasourcePanelDragOption from './DatasourcePanelDragOption';
 import { DndItemType } from '../DndItemType';
-import { DndItemValue } from './types';
+import { DndItemValue, FlattenedItem, Folder } from './types';
-
-export type DataSourcePanelColumn = {
-  is_dttm?: boolean | null;
-  description?: string | null;
-  expression?: string | null;
-  is_certified?: number | null;
-  column_name?: string | null;
-  name?: string | null;
-  type?: string;
-};
-
-type Props = {
-  index: number;
-  style: CSSProperties;
-  data: {
-    metricSlice: Metric[];
-    columnSlice: DataSourcePanelColumn[];
-    totalMetrics: number;
-    totalColumns: number;
-    width: number;
-    showAllMetrics: boolean;
-    onShowAllMetricsChange: (showAll: boolean) => void;
-    showAllColumns: boolean;
-    onShowAllColumnsChange: (showAll: boolean) => void;
-    collapseMetrics: boolean;
-    onCollapseMetricsChange: (collapse: boolean) => void;
-    collapseColumns: boolean;
-    onCollapseColumnsChange: (collapse: boolean) => void;
-    hiddenMetricCount: number;
-    hiddenColumnCount: number;
-  };
-};
-
-export const DEFAULT_MAX_COLUMNS_LENGTH = 50;
-export const DEFAULT_MAX_METRICS_LENGTH = 50;
-export const ITEM_HEIGHT = 30;
-
-const Button = styled.button`
-  background: none;
-  border: none;
-  text-decoration: underline;
-  color: ${({ theme }) => theme.colors.primary.dark1};
-`;
-
-const ButtonContainer = styled.div`
-  text-align: center;
-  padding-top: 2px;
-`;
 
 const LabelWrapper = styled.div`
   ${({ theme }) => css`
+    color: ${theme.colors.grayscale.dark1};
     overflow: hidden;
     text-overflow: ellipsis;
     font-size: ${theme.typography.sizes.s}px;
     background-color: ${theme.colors.grayscale.light4};
     margin: ${theme.gridUnit * 2}px 0;
-    border-radius: 4px;
+    border-radius: ${theme.borderRadius}px;
     padding: 0 ${theme.gridUnit}px;
 
     &:first-of-type {
@@ -123,146 +76,95 @@ const SectionHeaderButton = styled.button`
   border: none;
   background: transparent;
   width: 100%;
-  padding-inline: 0px;
+  height: 100%;
+  padding-inline: 0;
 `;
 
 const SectionHeader = styled.span`
-  ${({ theme }) => `
+  ${({ theme }) => css`
+    color: ${theme.colors.grayscale.dark1};
     font-size: ${theme.typography.sizes.m}px;
+    font-weight: ${theme.typography.weights.medium};
     line-height: 1.3;
   `}
 `;
 
-const Box = styled.div`
+const Divider = styled.div`
-  ${({ theme }) => `
+  ${({ theme }) => css`
-    border: 1px ${theme.colors.grayscale.light4} solid;
+    height: 16px;
-    border-radius: ${theme.gridUnit}px;
+    border-bottom: 1px solid ${theme.colors.grayscale.light3};
-    font-size: ${theme.typography.sizes.s}px;
-    padding: ${theme.gridUnit}px;
-    color: ${theme.colors.grayscale.light1};
-    text-overflow: ellipsis;
-    white-space: nowrap;
-    overflow: hidden;
   `}
 `;
 
-const DatasourcePanelItem: FC<Props> = ({ index, style, data }) => {
+export interface DatasourcePanelItemProps {
-  const {
+  index: number;
-    metricSlice: _metricSlice,
+  style: CSSProperties;
-    columnSlice,
+  data: {
-    totalMetrics,
+    flattenedItems: FlattenedItem[];
-    totalColumns,
+    folderMap: Map<string, Folder>;
-    width,
+    width: number;
-    showAllMetrics,
+    onToggleCollapse: (folderId: string) => void;
-    onShowAllMetricsChange,
+  };
-    showAllColumns,
+}
-    onShowAllColumnsChange,
-    collapseMetrics,
-    onCollapseMetricsChange,
-    collapseColumns,
-    onCollapseColumnsChange,
-    hiddenMetricCount,
-    hiddenColumnCount,
-  } = data;
-  const metricSlice = collapseMetrics ? [] : _metricSlice;
 
-  const EXTRA_LINES = collapseMetrics ? 1 : 2;
+const DatasourcePanelItem = ({
-  const isColumnSection = collapseMetrics
+  index,
-    ? index >= 1
+  style,
-    : index > metricSlice.length + EXTRA_LINES;
+  data,
-  const HEADER_LINE = isColumnSection
+}: DatasourcePanelItemProps) => {
-    ? metricSlice.length + EXTRA_LINES + 1
+  const { flattenedItems, folderMap, width, onToggleCollapse } = data;
-    : 0;
+  const item = flattenedItems[index];
-  const SUBTITLE_LINE = HEADER_LINE + 1;
-  const BOTTOM_LINE =
-    (isColumnSection ? columnSlice.length : metricSlice.length) +
-    (collapseMetrics ? HEADER_LINE : SUBTITLE_LINE) +
-    1;
-  const collapsed = isColumnSection ? collapseColumns : collapseMetrics;
-  const setCollapse = isColumnSection
-    ? onCollapseColumnsChange
-    : onCollapseMetricsChange;
-  const showAll = isColumnSection ? showAllColumns : showAllMetrics;
-  const setShowAll = isColumnSection
-    ? onShowAllColumnsChange
-    : onShowAllMetricsChange;
   const theme = useTheme();
-  const hiddenCount = isColumnSection ? hiddenColumnCount : hiddenMetricCount;
+
+  if (!item) return null;
+
+  const folder = folderMap.get(item.folderId);
+  if (!folder) return null;
+
+  const indentation = item.depth * theme.gridUnit * 4;
 
   return (
     <div
-      style={style}
+      style={{
-      css={css`
+        ...style,
-        padding: 0 ${theme.gridUnit * 4}px;
+        paddingLeft: theme.gridUnit * 4 + indentation,
-      `}
+        paddingRight: theme.gridUnit * 4,
+      }}
     >
-      {index === HEADER_LINE && (
+      {item.type === 'header' && (
-        <SectionHeaderButton onClick={() => setCollapse(!collapsed)}>
+        <SectionHeaderButton onClick={() => onToggleCollapse(folder.id)}>
-          <SectionHeader>
+          <SectionHeader>{folder.name}</SectionHeader>
-            {isColumnSection ? t('Columns') : t('Metrics')}
+          {folder.isCollapsed ? (
-          </SectionHeader>
-          {collapsed ? (
             <Icons.DownOutlined iconSize="s" />
           ) : (
             <Icons.UpOutlined iconSize="s" />
           )}
         </SectionHeaderButton>
       )}
-      {index === SUBTITLE_LINE && !collapsed && (
+
-        <div
+      {item.type === 'item' && item.item && (
-          css={css`
-            display: flex;
-            gap: ${theme.gridUnit * 2}px;
-            justify-content: space-between;
-            align-items: baseline;
-          `}
-        >
-          <div
-            className="field-length"
-            css={css`
-              flex-shrink: 0;
-            `}
-          >
-            {isColumnSection
-              ? t(`Showing %s of %s`, columnSlice?.length, totalColumns)
-              : t(`Showing %s of %s`, metricSlice?.length, totalMetrics)}
-          </div>
-          {hiddenCount > 0 && (
-            <Box>{t(`%s ineligible item(s) are hidden`, hiddenCount)}</Box>
-          )}
-        </div>
-      )}
-      {index > SUBTITLE_LINE && index < BOTTOM_LINE && (
         <LabelWrapper
           key={
-            (isColumnSection
+            (item.item.type === 'column'
-              ? columnSlice[index - SUBTITLE_LINE - 1].column_name
+              ? item.item.column_name
-              : metricSlice[index - SUBTITLE_LINE - 1].metric_name) +
+              : item.item.metric_name) + String(width)
-            String(width)
           }
           className="column"
         >
           <DatasourcePanelDragOption
-            value={
+            value={item.item as DndItemValue}
-              isColumnSection
+            type={
-                ? (columnSlice[index - SUBTITLE_LINE - 1] as DndItemValue)
+              item.item.type === 'column'
-                : metricSlice[index - SUBTITLE_LINE - 1]
+                ? DndItemType.Column
+                : DndItemType.Metric
             }
-            type={isColumnSection ? DndItemType.Column : DndItemType.Metric}
           />
         </LabelWrapper>
       )}
-      {index === BOTTOM_LINE &&
+
-        !collapsed &&
+      {item.type === 'divider' && (
-        (isColumnSection
+        <Divider data-test="datasource-panel-divider" />
-          ? totalColumns > DEFAULT_MAX_COLUMNS_LENGTH
+      )}
-          : totalMetrics > DEFAULT_MAX_METRICS_LENGTH) && (
-          <ButtonContainer>
-            <Button onClick={() => setShowAll(!showAll)}>
-              {showAll ? t('Show less...') : t('Show all...')}
-            </Button>
-          </ButtonContainer>
-        )}
     </div>
   );
 };

superset-frontend/src/explore/components/DatasourcePanel/fixtures.tsx

@@ -26,6 +26,7 @@ export const columns = [
     filterable: true,
     groupby: true,
     id: 516,
+    uuid: '516',
     is_dttm: false,
     python_date_format: null,
     type: 'DOUBLE',
@@ -40,6 +41,7 @@ export const columns = [
     filterable: true,
     groupby: true,
     id: 477,
+    uuid: '477',
     is_dttm: false,
     python_date_format: null,
     type: 'VARCHAR',
@@ -52,7 +54,8 @@ export const columns = [
     expression: null,
     filterable: true,
     groupby: true,
-    id: 516,
+    id: 517,
+    uuid: '517',
     is_dttm: false,
     python_date_format: null,
     type: 'INT',
@@ -70,6 +73,7 @@ const metricsFiltered = {
       description: null,
       expression: '',
       id: 56,
+      uuid: '56',
       is_certified: true,
       metric_name: 'metric_end_certified',
       verbose_name: '',
@@ -84,6 +88,7 @@ const metricsFiltered = {
       description: null,
       expression: '',
       id: 57,
+      uuid: '57',
       is_certified: false,
       metric_name: 'metric_end',
       verbose_name: '',

superset-frontend/src/explore/components/DatasourcePanel/index.tsx

@@ -28,7 +28,6 @@ import {
 
 import { ControlConfig } from '@superset-ui/chart-controls';
 import AutoSizer from 'react-virtualized-auto-sizer';
-import { FixedSizeList as List } from 'react-window';
 
 import { matchSorter, rankings } from 'match-sorter';
 import Alert from 'src/components/Alert';
@@ -39,22 +38,19 @@ import { FAST_DEBOUNCE } from 'src/constants';
 import { ExploreActions } from 'src/explore/actions/exploreActions';
 import Control from 'src/explore/components/Control';
 import { useDebounceValue } from 'src/hooks/useDebounceValue';
-import DatasourcePanelItem, {
-  ITEM_HEIGHT,
-  DataSourcePanelColumn,
-  DEFAULT_MAX_COLUMNS_LENGTH,
-  DEFAULT_MAX_METRICS_LENGTH,
-} from './DatasourcePanelItem';
 import { DndItemType } from '../DndItemType';
-import { DndItemValue } from './types';
+import { DatasourceFolder, DatasourcePanelColumn, DndItemValue } from './types';
 import { DropzoneContext } from '../ExploreContainer';
+import { DatasourceItems } from './DatasourceItems';
+import { transformDatasourceWithFolders } from './transformDatasourceFolders';
 
 interface DatasourceControl extends Omit<ControlConfig, 'hidden'> {
   datasource?: IDatasource;
 }
 export interface IDatasource {
   metrics: Metric[];
-  columns: DataSourcePanelColumn[];
+  columns: DatasourcePanelColumn[];
+  folders?: DatasourceFolder[];
   id: number;
   type: DatasourceType;
   database: {
@@ -126,8 +122,18 @@ const StyledInfoboxWrapper = styled.div`
 
 const BORDER_WIDTH = 2;
 
-const sortCertifiedFirst = (slice: DataSourcePanelColumn[]) =>
+const sortColumns = (slice: DatasourcePanelColumn[]) =>
-  slice.sort((a, b) => (b?.is_certified ?? 0) - (a?.is_certified ?? 0));
+  [...slice]
+    .sort((col1, col2) => {
+      if (col1?.is_dttm && !col2?.is_dttm) {
+        return -1;
+      }
+      if (col2?.is_dttm && !col1?.is_dttm) {
+        return 1;
+      }
+      return 0;
+    })
+    .sort((a, b) => (b?.is_certified ?? 0) - (a?.is_certified ?? 0));
 
 export default function DataSourcePanel({
   datasource,
@@ -137,7 +143,7 @@ export default function DataSourcePanel({
   width,
 }: Props) {
   const [dropzones] = useContext(DropzoneContext);
-  const { columns: _columns, metrics } = datasource;
+  const { columns: _columns, metrics, folders: _folders } = datasource;
 
   const allowedColumns = useMemo(() => {
     const validators = Object.values(dropzones);
@@ -152,21 +158,6 @@ export default function DataSourcePanel({
     );
   }, [dropzones, _columns]);
 
-  // display temporal column first
-  const columns = useMemo(
-    () =>
-      [...allowedColumns].sort((col1, col2) => {
-        if (col1?.is_dttm && !col2?.is_dttm) {
-          return -1;
-        }
-        if (col2?.is_dttm && !col1?.is_dttm) {
-          return 1;
-        }
-        return 0;
-      }),
-    [allowedColumns],
-  );
-
   const allowedMetrics = useMemo(() => {
     const validators = Object.values(dropzones);
     return metrics.filter(metric =>
@@ -176,21 +167,15 @@ export default function DataSourcePanel({
     );
   }, [dropzones, metrics]);
 
-  const hiddenColumnCount = _columns.length - allowedColumns.length;
-  const hiddenMetricCount = metrics.length - allowedMetrics.length;
   const [showSaveDatasetModal, setShowSaveDatasetModal] = useState(false);
   const [inputValue, setInputValue] = useState('');
-  const [showAllMetrics, setShowAllMetrics] = useState(false);
-  const [showAllColumns, setShowAllColumns] = useState(false);
-  const [collapseMetrics, setCollapseMetrics] = useState(false);
-  const [collapseColumns, setCollapseColumns] = useState(false);
   const searchKeyword = useDebounceValue(inputValue, FAST_DEBOUNCE);
 
   const filteredColumns = useMemo(() => {
     if (!searchKeyword) {
-      return columns ?? [];
+      return allowedColumns ?? [];
     }
-    return matchSorter(columns, searchKeyword, {
+    return matchSorter(allowedColumns, searchKeyword, {
       keys: [
         {
           key: 'verbose_name',
@@ -211,7 +196,7 @@ export default function DataSourcePanel({
       ],
       keepDiacritics: true,
     });
-  }, [columns, searchKeyword]);
+  }, [allowedColumns, searchKeyword]);
 
   const filteredMetrics = useMemo(() => {
     if (!searchKeyword) {
@@ -244,22 +229,15 @@ export default function DataSourcePanel({
     });
   }, [allowedMetrics, searchKeyword]);
 
-  const metricSlice = useMemo(
+  const sortedColumns = useMemo(
-    () =>
+    () => sortColumns(filteredColumns),
-      showAllMetrics
+    [filteredColumns],
-        ? filteredMetrics
-        : filteredMetrics?.slice?.(0, DEFAULT_MAX_METRICS_LENGTH),
-    [filteredMetrics, showAllMetrics],
   );
 
-  const columnSlice = useMemo(
+  const folders = useMemo(
     () =>
-      showAllColumns
+      transformDatasourceWithFolders(filteredMetrics, sortedColumns, _folders),
-        ? sortCertifiedFirst(filteredColumns)
+    [_folders, filteredMetrics, sortedColumns],
-        : sortCertifiedFirst(
-            filteredColumns?.slice?.(0, DEFAULT_MAX_COLUMNS_LENGTH),
-          ),
-    [filteredColumns, showAllColumns],
   );
 
   const showInfoboxCheck = () => {
@@ -324,57 +302,17 @@ export default function DataSourcePanel({
           )}
           <AutoSizer>
             {({ height }: { height: number }) => (
-              <List
+              <DatasourceItems
                 width={width - BORDER_WIDTH}
                 height={height}
-                itemSize={ITEM_HEIGHT}
+                folders={folders}
-                itemCount={
+              />
-                  (collapseMetrics ? 0 : metricSlice?.length) +
-                  (collapseColumns ? 0 : columnSlice.length) +
-                  2 + // Each section header row
-                  (collapseMetrics ? 0 : 2) +
-                  (collapseColumns ? 0 : 2)
-                }
-                itemData={{
-                  metricSlice,
-                  columnSlice,
-                  width,
-                  totalMetrics: filteredMetrics.length,
-                  totalColumns: filteredColumns.length,
-                  showAllMetrics,
-                  onShowAllMetricsChange: setShowAllMetrics,
-                  showAllColumns,
-                  onShowAllColumnsChange: setShowAllColumns,
-                  collapseMetrics,
-                  onCollapseMetricsChange: setCollapseMetrics,
-                  collapseColumns,
-                  onCollapseColumnsChange: setCollapseColumns,
-                  hiddenMetricCount,
-                  hiddenColumnCount,
-                }}
-                overscanCount={5}
-              >
-                {DatasourcePanelItem}
-              </List>
             )}
           </AutoSizer>
         </div>
       </>
     ),
-    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [inputValue, datasourceIsSaveable, width, folders],
-    [
-      columnSlice,
-      inputValue,
-      filteredColumns.length,
-      filteredMetrics.length,
-      metricSlice,
-      showAllColumns,
-      showAllMetrics,
-      collapseMetrics,
-      collapseColumns,
-      datasourceIsSaveable,
-      width,
-    ],
   );
 
   return (

superset-frontend/src/explore/components/DatasourcePanel/transformDatasourceFolders.test.ts

@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Metric } from '@superset-ui/core';
+import { transformDatasourceWithFolders } from './transformDatasourceFolders';
+import { DatasourceFolder, DatasourcePanelColumn } from './types';
+
+const mockMetrics: Metric[] = [
+  { metric_name: 'metric1', uuid: 'metric1-uuid', expression: 'SUM(col1)' },
+  { metric_name: 'metric2', uuid: 'metric2-uuid', expression: 'AVG(col2)' },
+  { metric_name: 'metric3', uuid: 'metric3-uuid', expression: 'COUNT(*)' },
+];
+
+const mockColumns: DatasourcePanelColumn[] = [
+  { column_name: 'column1', uuid: 'column1-uuid', type: 'STRING' },
+  { column_name: 'column2', uuid: 'column2-uuid', type: 'NUMERIC' },
+  { column_name: 'column3', uuid: 'column3-uuid', type: 'DATETIME' },
+];
+
+test('transforms data into default folders when no folder config is provided', () => {
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    undefined,
+  );
+
+  expect(result).toHaveLength(2);
+
+  expect(result[0].id).toBe('metrics-default');
+  expect(result[0].name).toBe('Metrics');
+  expect(result[0].items).toHaveLength(3);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+  expect(result[0].items[0].type).toBe('metric');
+
+  expect(result[1].id).toBe('columns-default');
+  expect(result[1].name).toBe('Columns');
+  expect(result[1].items).toHaveLength(3);
+  expect(result[1].items[0].uuid).toBe('column1-uuid');
+  expect(result[1].items[0].type).toBe('column');
+});
+
+test('transforms data according to folder configuration', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'folder1',
+      type: 'folder',
+      name: 'Important Metrics',
+      description: 'Key metrics folder',
+      children: [
+        { type: 'metric', uuid: 'metric1-uuid', name: 'metric1' },
+        { type: 'metric', uuid: 'metric2-uuid', name: 'metric2' },
+      ],
+    },
+    {
+      uuid: 'folder2',
+      type: 'folder',
+      name: 'Key Dimensions',
+      children: [{ type: 'column', uuid: 'column1-uuid', name: 'column1' }],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+  );
+
+  // We expect 4 folders:
+  // 1. Important Metrics (from config)
+  // 2. Key Dimensions (from config)
+  // 3. Metrics (default for unassigned metrics)
+  // 4. Columns (default for unassigned columns)
+  expect(result).toHaveLength(4);
+
+  expect(result[0].id).toBe('folder1');
+  expect(result[0].name).toBe('Important Metrics');
+  expect(result[0].description).toBe('Key metrics folder');
+  expect(result[0].items).toHaveLength(2);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+
+  expect(result[1].id).toBe('folder2');
+  expect(result[1].name).toBe('Key Dimensions');
+  expect(result[1].items).toHaveLength(1);
+  expect(result[1].items[0].uuid).toBe('column1-uuid');
+
+  expect(result[2].id).toBe('metrics-default');
+  expect(result[2].items).toHaveLength(1);
+  expect(result[2].items[0].uuid).toBe('metric3-uuid');
+
+  expect(result[3].id).toBe('columns-default');
+  expect(result[3].items).toHaveLength(2);
+});
+
+test('handles nested folder structures', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'parent-folder',
+      type: 'folder',
+      name: 'Parent Folder',
+      children: [
+        {
+          uuid: 'child-folder',
+          type: 'folder',
+          name: 'Child Folder',
+          children: [{ type: 'metric', uuid: 'metric1-uuid', name: 'metric1' }],
+        },
+        { type: 'column', uuid: 'column1-uuid', name: 'column1' },
+      ],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+  );
+
+  expect(result[0].id).toBe('parent-folder');
+  expect(result[0].name).toBe('Parent Folder');
+  expect(result[0].items).toHaveLength(1);
+  expect(result[0].subFolders).toHaveLength(1);
+
+  const childFolder = result[0].subFolders![0];
+  expect(childFolder.id).toBe('child-folder');
+  expect(childFolder.name).toBe('Child Folder');
+  expect(childFolder.items).toHaveLength(1);
+  expect(childFolder.parentId).toBe('parent-folder');
+});
+
+test('handles empty children arrays', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'empty-folder',
+      type: 'folder',
+      name: 'Empty Folder',
+      children: [],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+  );
+
+  expect(result[0].id).toBe('empty-folder');
+  expect(result[0].name).toBe('Empty Folder');
+  expect(result[0].items).toHaveLength(0);
+});
+
+test('handles non-existent metric and column UUIDs in folder config', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'folder1',
+      type: 'folder',
+      name: 'Test Folder',
+      children: [
+        {
+          type: 'metric',
+          uuid: 'non-existent-metric',
+          name: 'Missing Metric',
+        },
+        {
+          type: 'column',
+          uuid: 'non-existent-column',
+          name: 'Missing Column',
+        },
+        { type: 'metric', uuid: 'metric1-uuid', name: 'Existing Metric' },
+      ],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+  );
+
+  expect(result[0].id).toBe('folder1');
+  expect(result[0].items).toHaveLength(1);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+});

superset-frontend/src/explore/components/DatasourcePanel/transformDatasourceFolders.ts

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { Metric, t } from '@superset-ui/core';
+import {
+  ColumnItem,
+  DatasourceFolder,
+  DatasourcePanelColumn,
+  Folder,
+  MetricItem,
+} from './types';
+
+const transformToFolderStructure = (
+  metrics: MetricItem[],
+  columns: ColumnItem[],
+  folderConfig: DatasourceFolder[] | undefined,
+): Folder[] => {
+  const metricsMap = new Map<string, MetricItem>();
+  const columnsMap = new Map<string, ColumnItem>();
+
+  const assignedMetricUuids = new Set<string>();
+  const assignedColumnUuids = new Set<string>();
+
+  metrics.forEach(metric => {
+    metricsMap.set(metric.uuid, metric);
+  });
+
+  columns.forEach(column => {
+    columnsMap.set(column.uuid, column);
+  });
+
+  const processFolder = (
+    datasourceFolder: DatasourceFolder,
+    parentId?: string,
+  ): Folder => {
+    const folder: Folder = {
+      id: datasourceFolder.uuid,
+      name: datasourceFolder.name,
+      description: datasourceFolder.description,
+      isCollapsed: false,
+      items: [],
+      parentId,
+    };
+
+    if (datasourceFolder.children && datasourceFolder.children.length > 0) {
+      if (!folder.subFolders) {
+        folder.subFolders = [];
+      }
+
+      datasourceFolder.children.forEach(child => {
+        if (child.type === 'folder') {
+          folder.subFolders!.push(
+            processFolder(child as DatasourceFolder, folder.id),
+          );
+        } else if (child.type === 'metric') {
+          const metric = metricsMap.get(child.uuid);
+          if (metric) {
+            folder.items.push(metric);
+            assignedMetricUuids.add(metric.uuid);
+          }
+        } else if (child.type === 'column') {
+          const column = columnsMap.get(child.uuid);
+          if (column) {
+            folder.items.push(column);
+            assignedColumnUuids.add(column.uuid);
+          }
+        }
+      });
+    }
+
+    return folder;
+  };
+
+  if (!folderConfig) {
+    return [
+      {
+        id: 'metrics-default',
+        name: t('Metrics'),
+        isCollapsed: false,
+        items: metrics,
+      },
+      {
+        id: 'columns-default',
+        name: t('Columns'),
+        isCollapsed: false,
+        items: columns,
+      },
+    ];
+  }
+
+  const folders = folderConfig.map(config => processFolder(config));
+
+  const unassignedMetrics = metrics.filter(
+    metric => !assignedMetricUuids.has(metric.uuid),
+  );
+  const unassignedColumns = columns.filter(
+    column => !assignedColumnUuids.has(column.uuid),
+  );
+
+  if (unassignedMetrics.length > 0) {
+    folders.push({
+      id: 'metrics-default',
+      name: t('Metrics'),
+      isCollapsed: false,
+      items: unassignedMetrics,
+    });
+  }
+
+  if (unassignedColumns.length > 0) {
+    folders.push({
+      id: 'columns-default',
+      name: t('Columns'),
+      isCollapsed: false,
+      items: unassignedColumns,
+    });
+  }
+
+  return folders;
+};
+
+export const transformDatasourceWithFolders = (
+  metrics: Metric[],
+  columns: DatasourcePanelColumn[],
+  folderConfig: DatasourceFolder[] | undefined,
+): Folder[] => {
+  const metricsWithType: MetricItem[] = metrics.map(metric => ({
+    ...metric,
+    type: 'metric',
+  }));
+  const columnsWithType: ColumnItem[] = columns.map(column => ({
+    ...column,
+    type: 'column',
+  }));
+
+  return transformToFolderStructure(
+    metricsWithType,
+    columnsWithType,
+    folderConfig,
+  );
+};

superset-frontend/src/explore/components/DatasourcePanel/types.ts

@@ -35,3 +35,55 @@ export function isDatasourcePanelDndItem(
 export function isSavedMetric(item: any): item is Metric {
   return item?.metric_name;
 }
+
+export type DatasourcePanelColumn = {
+  uuid: string;
+  id?: number;
+  is_dttm?: boolean | null;
+  description?: string | null;
+  expression?: string | null;
+  is_certified?: number | null;
+  column_name?: string | null;
+  name?: string | null;
+  type?: string;
+};
+
+export type DatasourceFolder = {
+  uuid: string;
+  type: 'folder';
+  name: string;
+  description?: string;
+  children?: (
+    | DatasourceFolder
+    | { type: 'metric'; uuid: string; name: string }
+    | { type: 'column'; uuid: string; name: string }
+  )[];
+};
+
+export type MetricItem = Metric & {
+  type: 'metric';
+};
+
+export type ColumnItem = DatasourcePanelColumn & {
+  type: 'column';
+};
+
+export type FolderItem = MetricItem | ColumnItem;
+
+export interface Folder {
+  id: string;
+  name: string;
+  description?: string;
+  isCollapsed: boolean;
+  items: FolderItem[];
+  subFolders?: Folder[];
+  parentId?: string;
+}
+
+export interface FlattenedItem {
+  type: 'header' | 'item' | 'divider';
+  folderId: string;
+  depth: number;
+  item?: FolderItem;
+  height: number;
+}

superset-frontend/src/explore/components/ExploreContainer/ExploreContainer.test.tsx

@@ -76,7 +76,7 @@ test('should only propagate dragging state when dragging the panel option', () =
   const { container, getByText } = render(
     <ExploreContainer>
       <DatasourcePanelDragOption
-        value={{ metric_name: 'panel option' }}
+        value={{ metric_name: 'panel option', uuid: '1' }}
         type={DndItemType.Metric}
       />
       <OptionControlLabel

superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndFilterSelect.test.tsx

@@ -202,7 +202,11 @@ test('cannot drop a column that is not part of the simple column selection', ()
         type={DndItemType.Column}
       />
       <DatasourcePanelDragOption
-        value={{ metric_name: 'metric_a', expression: 'AGG(metric_a)' }}
+        value={{
+          metric_name: 'metric_a',
+          expression: 'AGG(metric_a)',
+          uuid: '1',
+        }}
         type={DndItemType.Metric}
       />
       {setup({
@@ -377,11 +381,11 @@ describe('when disallow_adhoc_metrics is set', () => {
           type={DndItemType.Column}
         />
         <DatasourcePanelDragOption
-          value={{ metric_name: 'metric_a' }}
+          value={{ metric_name: 'metric_a', uuid: '1' }}
           type={DndItemType.Metric}
         />
         <DatasourcePanelDragOption
-          value={{ metric_name: 'avg__num' }}
+          value={{ metric_name: 'avg__num', uuid: '2' }}
           type={DndItemType.AdhocMetricOption}
         />
         {setup({

superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndMetricSelect.test.tsx

@@ -334,7 +334,7 @@ test('cannot drop a duplicated item', () => {
   const { getByTestId } = render(
     <>
       <DatasourcePanelDragOption
-        value={{ metric_name: 'metric_a' }}
+        value={{ metric_name: 'metric_a', uuid: '1' }}
         type={DndItemType.Metric}
       />
       <DndMetricSelect {...defaultProps} value={metricValues} multi />
@@ -362,7 +362,7 @@ test('can drop a saved metric when disallow_adhoc_metrics', () => {
   const { getByTestId } = render(
     <>
       <DatasourcePanelDragOption
-        value={{ metric_name: 'metric_a' }}
+        value={{ metric_name: 'metric_a', uuid: '1' }}
         type={DndItemType.Metric}
       />
       <DndMetricSelect
@@ -395,15 +395,15 @@ test('cannot drop non-saved metrics when disallow_adhoc_metrics', () => {
   const { getByTestId, getAllByTestId } = render(
     <>
       <DatasourcePanelDragOption
-        value={{ metric_name: 'metric_a' }}
+        value={{ metric_name: 'metric_a', uuid: '1' }}
         type={DndItemType.Metric}
       />
       <DatasourcePanelDragOption
-        value={{ metric_name: 'metric_c' }}
+        value={{ metric_name: 'metric_c', uuid: '2' }}
         type={DndItemType.Metric}
       />
       <DatasourcePanelDragOption
-        value={{ column_name: 'column_1' }}
+        value={{ column_name: 'column_1', uuid: '3' }}
         type={DndItemType.Column}
       />
       <DndMetricSelect

superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndMetricSelect.tsx

@@ -18,6 +18,7 @@
  */
 
 import { useCallback, useEffect, useMemo, useState } from 'react';
+import { nanoid } from 'nanoid';
 import {
   ensureIsArray,
   GenericDataType,
@@ -77,6 +78,7 @@ const coerceMetrics = (
       return {
         metric_name: metric,
         error_text: t('This metric might be incompatible with current dataset'),
+        uuid: nanoid(),
       };
     }
     if (!isDictionaryForAdhocMetric(metric)) {

superset-frontend/src/explore/controlUtils/controlUtils.test.tsx

@@ -51,7 +51,10 @@ describe('controlUtils', () => {
       id: 1,
       type: DatasourceType.Table,
       columns: [{ column_name: 'a' }],
-      metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
+      metrics: [
+        { metric_name: 'first', uuid: '1' },
+        { metric_name: 'second', uuid: '2' },
+      ],
       column_formats: {},
       currency_formats: {},
       verbose_map: {},

superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.test.ts

@@ -33,7 +33,7 @@ const sampleDatasource: Dataset = {
     { column_name: 'sample_column_3' },
     { column_name: 'sample_column_4' },
   ],
-  metrics: [{ metric_name: 'saved_metric_2' }],
+  metrics: [{ metric_name: 'saved_metric_2', uuid: '1' }],
   column_formats: {},
   currency_formats: {},
   verbose_map: {},

superset-frontend/src/explore/fixtures.tsx

@@ -133,7 +133,10 @@ export const exploreInitialData: ExplorePageInitialData = {
     id: 8,
     type: DatasourceType.Table,
     columns: [{ column_name: 'a' }],
-    metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
+    metrics: [
+      { metric_name: 'first', uuid: '1' },
+      { metric_name: 'second', uuid: '2' },
+    ],
     column_formats: {},
     currency_formats: {},
     verbose_map: {},

superset-frontend/src/features/datasets/types.ts

@@ -41,7 +41,7 @@ export type ColumnObject = {
 
 type MetricObject = {
   id: number;
-  uuid: number;
+  uuid: string;
   expression?: string;
   description?: string;
   metric_name: string;

superset/commands/dataset/update.py

@@ -17,13 +17,13 @@
 import logging
 from collections import Counter
 from functools import partial
-from typing import Any, Optional
+from typing import Any, cast, Optional
 
 from flask_appbuilder.models.sqla import Model
 from marshmallow import ValidationError
 from sqlalchemy.exc import SQLAlchemyError
 
-from superset import security_manager
+from superset import is_feature_enabled, security_manager
 from superset.commands.base import BaseCommand, UpdateMixin
 from superset.commands.dataset.exceptions import (
     DatabaseChangeValidationError,
@@ -39,8 +39,9 @@ from superset.commands.dataset.exceptions import (
     DatasetNotFoundError,
     DatasetUpdateFailedError,
 )
-from superset.connectors.sqla.models import SqlaTable
+from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn
 from superset.daos.dataset import DatasetDAO
+from superset.datasets.schemas import FolderSchema
 from superset.exceptions import SupersetSecurityException
 from superset.sql_parse import Table
 from superset.utils.decorators import on_error, transaction
@@ -127,17 +128,31 @@ class UpdateDatasetCommand(UpdateMixin, BaseCommand):
         except ValidationError as ex:
             exceptions.append(ex)
 
-        # Validate columns
+        self._validate_semantics(exceptions)
-        if columns := self._properties.get("columns"):
-            self._validate_columns(columns, exceptions)
-
-        # Validate metrics
-        if metrics := self._properties.get("metrics"):
-            self._validate_metrics(metrics, exceptions)
 
         if exceptions:
             raise DatasetInvalidError(exceptions=exceptions)
 
+    def _validate_semantics(self, exceptions: list[ValidationError]) -> None:
+        # we know we have a valid model
+        self._model = cast(SqlaTable, self._model)
+
+        if columns := self._properties.get("columns"):
+            self._validate_columns(columns, exceptions)
+
+        if metrics := self._properties.get("metrics"):
+            self._validate_metrics(metrics, exceptions)
+
+        if folders := self._properties.get("folders"):
+            try:
+                validate_folders(folders, self._model.metrics, self._model.columns)
+            except ValidationError as ex:
+                exceptions.append(ex)
+
+            # dump schema to convert UUID to string
+            schema = FolderSchema(many=True)
+            self._properties["folders"] = schema.dump(folders)
+
     def _validate_columns(
         self, columns: list[dict[str, Any]], exceptions: list[ValidationError]
     ) -> None:
@@ -189,3 +204,60 @@ class UpdateDatasetCommand(UpdateMixin, BaseCommand):
             if count > 1
         ]
         return duplicates
+
+
+def validate_folders(  # noqa: C901
+    folders: list[FolderSchema],
+    metrics: list[SqlMetric],
+    columns: list[TableColumn],
+) -> None:
+    """
+    Additional folder validation.
+
+    The marshmallow schema will validate the folder structure, but we still need to
+    check that UUIDs are valid, names are unique and not reserved, and that there are
+    no cycles.
+    """
+    if not is_feature_enabled("DATASET_FOLDERS"):
+        raise ValidationError("Dataset folders are not enabled")
+
+    existing = {
+        "metric": {metric.uuid: metric.metric_name for metric in metrics},
+        "column": {column.uuid: column.column_name for column in columns},
+    }
+
+    queue: list[tuple[FolderSchema, list[str]]] = [(folder, []) for folder in folders]
+    seen_uuids = set()
+    seen_fqns = set()  # fully qualified names
+    while queue:
+        obj, path = queue.pop(0)
+        uuid, name, type = obj["uuid"], obj["name"], obj["type"]
+
+        if uuid in path:
+            raise ValidationError(f"Cycle detected: {uuid} appears in its ancestry")
+
+        if uuid in seen_uuids:
+            raise ValidationError(f"Duplicate UUID in folder structure: {uuid}")
+        seen_uuids.add(uuid)
+
+        # folders can have duplicate name as long as they're not siblings
+        fqn = tuple(path + [name])
+        if type == "folder" and fqn in seen_fqns:
+            raise ValidationError(f"Duplicate folder name: {name}")
+        seen_fqns.add(fqn)
+
+        if type == "folder" and name.lower() in {
+            "metrics",
+            "columns",
+        }:
+            raise ValidationError(f"Folder cannot have name '{name}'")
+
+        if type in {"metric", "column"}:
+            if uuid not in existing[type]:
+                raise ValidationError(f"Invalid UUID for {type} '{name}': {uuid}")
+            if name != existing[type][uuid]:
+                raise ValidationError(f"Mismatched name '{name}' for UUID '{uuid}'")
+
+        if children := obj.get("children"):
+            path.append(uuid)
+            queue.extend((folder, path) for folder in children)

superset/config.py

@@ -561,6 +561,9 @@ DEFAULT_FEATURE_FLAGS: dict[str, bool] = {
     "SLACK_ENABLE_AVATARS": False,
     # Allow users to optionally specify date formats in email subjects, which will be parsed if enabled. # noqa: E501
     "DATE_FORMAT_IN_EMAIL_SUBJECT": False,
+    # Allow metrics and columns to be grouped into (potentially nested) folders in the
+    # chart builder
+    "DATASET_FOLDERS": False,
 }
 
 # ------------------------------

superset/connectors/sqla/models.py

@@ -69,6 +69,7 @@ from sqlalchemy.sql import column, ColumnElement, literal_column, table
 from sqlalchemy.sql.elements import ColumnClause, TextClause
 from sqlalchemy.sql.expression import Label
 from sqlalchemy.sql.selectable import Alias, TableClause
+from sqlalchemy.types import JSON
 
 from superset import app, db, is_feature_enabled, security_manager
 from superset.commands.dataset.exceptions import DatasetNotFoundError
@@ -400,6 +401,7 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin):  # pylint: disable=
             # one to many
             "columns": [o.data for o in self.columns],
             "metrics": [o.data for o in self.metrics],
+            "folders": self.folders,
             # TODO deprecate, move logic to JS
             "order_by_choices": self.order_by_choices,
             "owners": [owner.id for owner in self.owners],
@@ -1018,6 +1020,7 @@ class TableColumn(AuditMixinNullable, ImportExportMixin, CertificationMixin, Mod
             "filterable",
             "groupby",
             "id",
+            "uuid",
             "is_certified",
             "is_dttm",
             "python_date_format",
@@ -1065,7 +1068,7 @@ class SqlMetric(AuditMixinNullable, ImportExportMixin, CertificationMixin, Model
         "extra",
         "warning_text",
     ]
-    update_from_object_fields = list(s for s in export_fields if s != "table_id")  # noqa: C400
+    update_from_object_fields = [s for s in export_fields if s != "table_id"]
     export_parent = "table"
 
     def __repr__(self) -> str:
@@ -1117,6 +1120,7 @@ class SqlMetric(AuditMixinNullable, ImportExportMixin, CertificationMixin, Model
             "description",
             "expression",
             "id",
+            "uuid",
             "is_certified",
             "metric_name",
             "warning_markdown",
@@ -1193,6 +1197,7 @@ class SqlaTable(
     extra = Column(Text)
     normalize_columns = Column(Boolean, default=False)
     always_filter_main_dttm = Column(Boolean, default=False)
+    folders = Column(JSON, nullable=True)
 
     baselink = "tablemodelview"
 

superset/datasets/api.py

@@ -194,8 +194,10 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         "metrics.id",
         "metrics.metric_name",
         "metrics.metric_type",
+        "metrics.uuid",
         "metrics.verbose_name",
         "metrics.warning_text",
+        "folders",
         "datasource_type",
         "url",
         "extra",
@@ -621,9 +623,11 @@ class DatasetRestApi(BaseSupersetModelRestApi):
             return self.response(201, id=new_model.id, result=item)
         except DatasetInvalidError as ex:
             return self.response_422(
-                message=ex.normalized_messages()
+                message=(
-                if isinstance(ex, ValidationError)
+                    ex.normalized_messages()
-                else str(ex)
+                    if isinstance(ex, ValidationError)
+                    else str(ex)
+                )
             )
         except DatasetCreateFailedError as ex:
             logger.error(
@@ -1176,14 +1180,16 @@ class DatasetRestApi(BaseSupersetModelRestApi):
 
         def render_item_list(item_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
             return [
-                {
+                (
-                    **item,
+                    {
-                    "rendered_expression": processor.process_template(
+                        **item,
-                        item["expression"]
+                        "rendered_expression": processor.process_template(
-                    ),
+                            item["expression"]
-                }
+                        ),
-                if item.get("expression")
+                    }
-                else item
+                    if item.get("expression")
+                    else item
+                )
                 for item in item_list
             ]
 

superset/datasets/schemas.py

@@ -20,7 +20,7 @@ from typing import Any
 from dateutil.parser import isoparse
 from flask_babel import lazy_gettext as _
 from marshmallow import fields, pre_load, Schema, ValidationError
-from marshmallow.validate import Length
+from marshmallow.validate import Length, OneOf
 
 from superset.exceptions import SupersetMarshmallowValidationError
 from superset.utils import json
@@ -88,6 +88,18 @@ class DatasetMetricsPutSchema(Schema):
     uuid = fields.UUID(allow_none=True)
 
 
+class FolderSchema(Schema):
+    uuid = fields.UUID()
+    type = fields.String(
+        required=False,
+        validate=OneOf(["metric", "column", "folder"]),
+    )
+    name = fields.String(required=True, validate=Length(1, 250))
+    description = fields.String(allow_none=True, validate=Length(0, 1000))
+    # folder can contain metrics, columns, and subfolders:
+    children = fields.List(fields.Nested(lambda: FolderSchema()), allow_none=True)
+
+
 class DatasetPostSchema(Schema):
     database = fields.Integer(required=True)
     catalog = fields.String(allow_none=True, validate=Length(0, 250))
@@ -121,6 +133,7 @@ class DatasetPutSchema(Schema):
     owners = fields.List(fields.Integer())
     columns = fields.List(fields.Nested(DatasetColumnsPutSchema))
     metrics = fields.List(fields.Nested(DatasetMetricsPutSchema))
+    folders = fields.List(fields.Nested(FolderSchema), required=False)
     extra = fields.String(allow_none=True)
     is_managed_externally = fields.Boolean(allow_none=True, dump_default=False)
     external_url = fields.String(allow_none=True)

superset/migrations/versions/2025-03-03_20-52_94e7a3499973_add_folder_table.py

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Add folder table
+
+Revision ID: 94e7a3499973
+Revises: 74ad1125881c
+Create Date: 2025-03-03 20:52:24.585143
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.types import JSON
+
+# revision identifiers, used by Alembic.
+revision = "94e7a3499973"
+down_revision = "74ad1125881c"
+
+
+def upgrade():
+    op.add_column(
+        "tables",
+        sa.Column("folders", JSON, nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("tables", "folders")

tests/integration_tests/datasets/api_tests.py

@@ -255,7 +255,7 @@ class TestDatasetApi(SupersetTestCase):
             "table_name",
             "uuid",
         ]
-        assert sorted(list(response["result"][0].keys())) == expected_columns  # noqa: C414
+        assert sorted(response["result"][0]) == expected_columns
 
     def test_get_dataset_list_gamma(self):
         """
@@ -1563,6 +1563,92 @@ class TestDatasetApi(SupersetTestCase):
         db.session.delete(dataset)
         db.session.commit()
 
+    @with_feature_flags(DATASET_FOLDERS=True)
+    def test_update_dataset_add_folders(self):
+        """
+        Dataset API: Test adding folders to dataset
+        """
+        self.login(username="admin")
+
+        dataset = self.insert_default_dataset()
+        dataset_data = {
+            "folders": [
+                {
+                    "type": "folder",
+                    "uuid": "b49ac3dd-c79b-42a4-9082-39ee74f3b369",
+                    "name": "My metrics",
+                    "children": [
+                        {
+                            "type": "metric",
+                            "uuid": dataset.metrics[0].uuid,
+                            "name": dataset.metrics[0].metric_name,
+                        },
+                    ],
+                },
+                {
+                    "type": "folder",
+                    "uuid": "f5db85fa-75d6-45e5-bdce-c6194db80642",
+                    "name": "My columns",
+                    "children": [
+                        {
+                            "type": "folder",
+                            "uuid": "b5330233-e323-4157-b767-98b16f00ca93",
+                            "name": "Dimensions",
+                            "children": [
+                                {
+                                    "type": "column",
+                                    "uuid": dataset.columns[1].uuid,
+                                    "name": dataset.columns[1].column_name,
+                                },
+                            ],
+                        },
+                    ],
+                },
+            ]
+        }
+
+        uri = f"api/v1/dataset/{dataset.id}"
+        rv = self.put_assert_metric(uri, dataset_data, "put")
+        assert rv.status_code == 200
+
+        model = db.session.query(SqlaTable).get(dataset.id)
+        assert model.folders == [
+            {
+                "uuid": "b49ac3dd-c79b-42a4-9082-39ee74f3b369",
+                "type": "folder",
+                "name": "My metrics",
+                "children": [
+                    {
+                        "uuid": str(dataset.metrics[0].uuid),
+                        "type": "metric",
+                        "name": "count",
+                    }
+                ],
+            },
+            {
+                "uuid": "f5db85fa-75d6-45e5-bdce-c6194db80642",
+                "type": "folder",
+                "name": "My columns",
+                "children": [
+                    {
+                        "uuid": "b5330233-e323-4157-b767-98b16f00ca93",
+                        "type": "folder",
+                        "name": "Dimensions",
+                        "children": [
+                            {
+                                "uuid": str(dataset.columns[1].uuid),
+                                "type": "column",
+                                "name": "name",
+                            }
+                        ],
+                    }
+                ],
+            },
+        ]
+
+        db.session.delete(dataset)
+        db.session.commit()
+
     def test_delete_dataset_item(self):
         """
         Dataset API: Test delete dataset item

tests/unit_tests/commands/dataset/test_update.py

@@ -14,16 +14,21 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+from typing import cast
 from unittest.mock import MagicMock
 
 import pytest
+from marshmallow import ValidationError
 from pytest_mock import MockerFixture
 
 from superset import db
 from superset.commands.dataset.exceptions import DatasetInvalidError
-from superset.commands.dataset.update import UpdateDatasetCommand
+from superset.commands.dataset.update import UpdateDatasetCommand, validate_folders
 from superset.connectors.sqla.models import SqlaTable
+from superset.datasets.schemas import FolderSchema
 from superset.models.core import Database
+from tests.unit_tests.conftest import with_feature_flags
 
 
 @pytest.mark.usefixture("session")
@@ -58,3 +63,350 @@ def test_update_uniqueness_error(mocker: MockerFixture) -> None:
                 "schema": "qux",
             },
         ).run()
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders(mocker: MockerFixture) -> None:
+    """
+    Test the folder validation.
+    """
+    metrics = [mocker.MagicMock(metric_name="metric1", uuid="uuid1")]
+    columns = [
+        mocker.MagicMock(column_name="column1", uuid="uuid2"),
+        mocker.MagicMock(column_name="column2", uuid="uuid3"),
+    ]
+
+    validate_folders(folders=[], metrics=metrics, columns=columns)
+
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid4",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid1",
+                        "type": "metric",
+                        "name": "metric1",
+                    },
+                    {
+                        "uuid": "uuid2",
+                        "type": "column",
+                        "name": "column1",
+                    },
+                    {
+                        "uuid": "uuid3",
+                        "type": "column",
+                        "name": "column2",
+                    },
+                ],
+            },
+        ],
+    )
+    validate_folders(folders=folders, metrics=metrics, columns=columns)
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_cycle(mocker: MockerFixture) -> None:
+    """
+    Test that we can detect cycles in the folder structure.
+    """
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "folder",
+                        "name": "My other folder",
+                        "children": [
+                            {
+                                "uuid": "uuid1",
+                                "type": "folder",
+                                "name": "My folder",
+                                "children": [],
+                            },
+                        ],
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=[], columns=[])
+    assert str(excinfo.value) == "Cycle detected: uuid1 appears in its ancestry"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_inter_cycle(mocker: MockerFixture) -> None:
+    """
+    Test that we can detect cycles between folders.
+    """
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "folder",
+                        "name": "My other folder",
+                        "children": [],
+                    },
+                ],
+            },
+            {
+                "uuid": "uuid2",
+                "type": "folder",
+                "name": "My other folder",
+                "children": [
+                    {
+                        "uuid": "uuid1",
+                        "type": "folder",
+                        "name": "My folder",
+                        "children": [],
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=[], columns=[])
+    assert str(excinfo.value) == "Duplicate UUID in folder structure: uuid2"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_duplicates(mocker: MockerFixture) -> None:
+    """
+    Test that metrics and columns belong to a single folder.
+    """
+    metrics = [mocker.MagicMock(metric_name="count", uuid="uuid2")]
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "metric",
+                        "name": "count",
+                    },
+                ],
+            },
+            {
+                "uuid": "uuid2",
+                "type": "folder",
+                "name": "My other folder",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "metric",
+                        "name": "count",
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=metrics, columns=[])
+    assert str(excinfo.value) == "Duplicate UUID in folder structure: uuid2"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_duplicate_name_not_siblings(mocker: MockerFixture) -> None:
+    """
+    Duplicate folder names are allowed if folders are not siblings.
+    """
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "Sales",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "folder",
+                        "name": "Core",
+                        "children": [],
+                    },
+                ],
+            },
+            {
+                "uuid": "uuid3",
+                "type": "folder",
+                "name": "Engineering",
+                "children": [
+                    {
+                        "uuid": "uuid4",
+                        "type": "folder",
+                        "name": "Core",
+                        "children": [],
+                    },
+                ],
+            },
+        ],
+    )
+
+    validate_folders(folders=folders, metrics=[], columns=[])
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_duplicate_name_siblings(mocker: MockerFixture) -> None:
+    """
+    Duplicate folder names are not allowed if folders are siblings.
+    """
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "Sales",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "folder",
+                        "name": "Core",
+                        "children": [],
+                    },
+                ],
+            },
+            {
+                "uuid": "uuid3",
+                "type": "folder",
+                "name": "Sales",
+                "children": [
+                    {
+                        "uuid": "uuid4",
+                        "type": "folder",
+                        "name": "Other",
+                        "children": [],
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=[], columns=[])
+    assert str(excinfo.value) == "Duplicate folder name: Sales"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_invalid_names(mocker: MockerFixture) -> None:
+    """
+    Test that we can detect reserved folder names.
+    """
+    folders_with_metrics = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "Metrics",
+                "children": [],
+            },
+        ],
+    )
+    folders_with_columns = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid1",
+                "type": "folder",
+                "name": "Columns",
+                "children": [],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders_with_metrics, metrics=[], columns=[])
+    assert str(excinfo.value) == "Folder cannot have name 'Metrics'"
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders_with_columns, metrics=[], columns=[])
+    assert str(excinfo.value) == "Folder cannot have name 'Columns'"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_invalid_uuid(mocker: MockerFixture) -> None:
+    """
+    Test that we can detect invalid UUIDs.
+    """
+    metrics = [mocker.MagicMock(metric_name="metric1", uuid="uuid1")]
+    columns = [
+        mocker.MagicMock(column_name="column1", uuid="uuid2"),
+        mocker.MagicMock(column_name="column2", uuid="uuid3"),
+    ]
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid4",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid2",
+                        "type": "metric",
+                        "name": "metric1",
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=metrics, columns=columns)
+    assert str(excinfo.value) == "Invalid UUID for metric 'metric1': uuid2"
+
+
+@with_feature_flags(DATASET_FOLDERS=True)
+def test_validate_folders_mismatched_name(mocker: MockerFixture) -> None:
+    """
+    Test that we can detect mismatched names.
+    """
+    metrics = [mocker.MagicMock(metric_name="metric1", uuid="uuid1")]
+    columns = [
+        mocker.MagicMock(column_name="column1", uuid="uuid2"),
+        mocker.MagicMock(column_name="column2", uuid="uuid3"),
+    ]
+    folders = cast(
+        list[FolderSchema],
+        [
+            {
+                "uuid": "uuid4",
+                "type": "folder",
+                "name": "My folder",
+                "children": [
+                    {
+                        "uuid": "uuid1",
+                        "type": "metric",
+                        "name": "metric2",
+                    },
+                ],
+            },
+        ],
+    )
+
+    with pytest.raises(ValidationError) as excinfo:
+        validate_folders(folders=folders, metrics=metrics, columns=columns)
+    assert str(excinfo.value) == "Mismatched name 'metric2' for UUID 'uuid1'"