feat(explore): Integrate dataset panel with Folders feature (#33104)

Co-authored-by: Kamil Gabryjelski <kamil.gabryjelski@gmail.com>

superset-frontend/src/explore/components/DatasourcePanel/DatasourceItems.tsx

@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { VariableSizeList as List } from 'react-window';
+import { FlattenedItem, Folder } from './types';
+import DatasourcePanelItem from './DatasourcePanelItem';
+
+const BORDER_WIDTH = 2;
+const HEADER_ITEM_HEIGHT = 50;
+const METRIC_OR_COLUMN_ITEM_HEIGHT = 32;
+const SUBTITLE_ITEM_HEIGHT = 32;
+const DIVIDER_ITEM_HEIGHT = 16;
+
+const flattenFolderStructure = (
+  foldersToFlatten: Folder[],
+  collapsedFolderIds: Set<string>,
+  depth = 0,
+  folderMap: Map<string, Folder> = new Map(),
+): { flattenedItems: FlattenedItem[]; folderMap: Map<string, Folder> } => {
+  const flattenedItems: FlattenedItem[] = [];
+
+  foldersToFlatten.forEach((folder, idx) => {
+    folderMap.set(folder.id, folder);
+
+    flattenedItems.push({
+      type: 'header',
+      folderId: folder.id,
+      depth,
+      height: HEADER_ITEM_HEIGHT,
+    });
+
+    if (!collapsedFolderIds.has(folder.id)) {
+      flattenedItems.push({
+        type: 'subtitle',
+        folderId: folder.id,
+        depth,
+        height: SUBTITLE_ITEM_HEIGHT,
+        totalItems: folder.totalItems,
+        showingItems: folder.showingItems,
+      });
+      folder.items.forEach(item => {
+        flattenedItems.push({
+          type: 'item',
+          folderId: folder.id,
+          depth,
+          item,
+          height: METRIC_OR_COLUMN_ITEM_HEIGHT,
+        });
+      });
+
+      if (folder.subFolders && folder.subFolders.length > 0) {
+        const { flattenedItems: subItems } = flattenFolderStructure(
+          folder.subFolders,
+          collapsedFolderIds,
+          depth + 1,
+          folderMap,
+        );
+
+        flattenedItems.push(...subItems);
+      }
+    }
+    if (depth === 0 && idx !== foldersToFlatten.length - 1) {
+      flattenedItems.push({
+        type: 'divider',
+        folderId: folder.id,
+        depth,
+        height: DIVIDER_ITEM_HEIGHT,
+      });
+    }
+  });
+
+  return { flattenedItems, folderMap };
+};
+
+interface DatasourceItemsProps {
+  width: number;
+  height: number;
+  folders: Folder[];
+}
+export const DatasourceItems = ({
+  width,
+  height,
+  folders,
+}: DatasourceItemsProps) => {
+  const listRef = useRef<List>(null);
+  const [collapsedFolderIds, setCollapsedFolderIds] = useState<Set<string>>(
+    new Set(
+      folders.filter(folder => folder.isCollapsed).map(folder => folder.id),
+    ),
+  );
+
+  const { flattenedItems, folderMap } = useMemo(
+    () => flattenFolderStructure(folders, collapsedFolderIds),
+    [folders, collapsedFolderIds],
+  );
+
+  const handleToggleCollapse = useCallback((folderId: string) => {
+    setCollapsedFolderIds(prevIds => {
+      const newIds = new Set(prevIds);
+      if (newIds.has(folderId)) {
+        newIds.delete(folderId);
+      } else {
+        newIds.add(folderId);
+      }
+      return newIds;
+    });
+  }, []);
+
+  useEffect(() => {
+    // reset the list cache when flattenedItems length changes to recalculate the heights
+    listRef.current?.resetAfterIndex(0);
+  }, [flattenedItems]);
+
+  const getItemSize = useCallback(
+    (index: number) => flattenedItems[index].height,
+    [flattenedItems],
+  );
+
+  const itemData = useMemo(
+    () => ({
+      flattenedItems,
+      folderMap,
+      width,
+      onToggleCollapse: handleToggleCollapse,
+      collapsedFolderIds,
+    }),
+    [
+      flattenedItems,
+      folderMap,
+      width,
+      handleToggleCollapse,
+      collapsedFolderIds,
+    ],
+  );
+
+  return (
+    <List
+      ref={listRef}
+      width={width - BORDER_WIDTH}
+      height={height}
+      itemSize={getItemSize}
+      itemCount={flattenedItems.length}
+      itemData={itemData}
+      overscanCount={5}
+    >
+      {DatasourcePanelItem}
+    </List>
+  );
+};

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanel.test.tsx

@@ -59,6 +59,48 @@ const datasource: IDatasource = {
   datasource_name: 'table1',
 };
 
+const datasourceWithFolders: IDatasource = {
+  ...datasource,
+  folders: [
+    {
+      name: 'Test folder',
+      type: 'folder',
+      uuid: '1',
+      children: [
+        {
+          name: 'Test nested folder',
+          type: 'folder',
+          uuid: '1.1',
+          children: [
+            {
+              type: 'metric',
+              uuid: metrics[0].uuid,
+              name: metrics[0].metric_name,
+            },
+          ],
+        },
+      ],
+    },
+    {
+      name: 'Second test folder',
+      type: 'folder',
+      uuid: '2',
+      children: [
+        {
+          type: 'column',
+          uuid: columns[0].uuid,
+          name: columns[0].column_name,
+        },
+        {
+          type: 'column',
+          uuid: columns[1].uuid,
+          name: columns[1].column_name,
+        },
+      ],
+    },
+  ],
+};
+
 const mockUser = {
   createdOn: '2021-04-27T18:12:38.952304',
   email: 'admin',
@@ -90,6 +132,18 @@ const props: DatasourcePanelProps = {
   width: 300,
 };
 
+const propsWithFolders = {
+  ...props,
+  datasource: datasourceWithFolders,
+  controls: {
+    ...props.controls,
+    datasource: {
+      ...props.controls.datasource,
+      datasource: datasourceWithFolders,
+    },
+  },
+};
+
 const metricProps = {
   savedMetrics: [],
   columns: [],
@@ -125,13 +179,9 @@ test('should render the metrics', async () => {
     </ExploreContainer>,
     { useRedux: true, useDnd: true },
   );
-  const metricsNum = metrics.length;
   metrics.forEach(metric =>
     expect(screen.getByText(metric.metric_name)).toBeInTheDocument(),
   );
-  expect(
-    await screen.findByText(`Showing ${metricsNum} of ${metricsNum}`),
-  ).toBeInTheDocument();
 });
 
 test('should render the columns', async () => {
@@ -142,13 +192,9 @@ test('should render the columns', async () => {
     </ExploreContainer>,
     { useRedux: true, useDnd: true },
   );
-  const columnsNum = columns.length;
   columns.forEach(col =>
     expect(screen.getByText(col.column_name)).toBeInTheDocument(),
   );
-  expect(
-    await screen.findByText(`Showing ${columnsNum} of ${columnsNum}`),
-  ).toBeInTheDocument();
 });
 
 describe('DatasourcePanel', () => {
@@ -310,3 +356,139 @@ test('should render only droppable metrics and columns', async () => {
 
   unmount();
 });
+
+test('Renders with custom folders', () => {
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  columns.forEach(col => {
+    expect(screen.getByText(col.column_name)).toBeInTheDocument();
+  });
+
+  metrics.forEach(metric => {
+    expect(screen.getByText(metric.metric_name)).toBeInTheDocument();
+  });
+
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
+  expect(screen.getAllByTestId('datasource-panel-divider').length).toEqual(3);
+});
+
+test('Collapse folders', () => {
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  userEvent.click(screen.getByText('Test folder'));
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.queryByText('Test nested folder')).not.toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  expect(screen.queryByText(metrics[0].metric_name)).not.toBeInTheDocument();
+
+  userEvent.click(screen.getByText('Test folder'));
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+
+  expect(screen.getByText(metrics[0].metric_name)).toBeInTheDocument();
+});
+
+test('Default Metrics and Columns folders dont render when all metrics and columns are assigned to custom folders', () => {
+  const datasourceWithFullFolders: IDatasource = {
+    ...datasource,
+    folders: [
+      {
+        name: 'Test folder',
+        type: 'folder',
+        uuid: '1',
+        children: [
+          {
+            name: 'Test nested folder',
+            type: 'folder',
+            uuid: '1.1',
+            children: metrics.map(m => ({
+              type: 'metric' as const,
+              uuid: m.uuid,
+              name: m.metric_name,
+            })),
+          },
+        ],
+      },
+      {
+        name: 'Second test folder',
+        type: 'folder',
+        uuid: '2',
+        children: columns.map(c => ({
+          type: 'column',
+          uuid: c.uuid,
+          name: c.column_name,
+        })),
+      },
+    ],
+  };
+  const propsWithFullFolders = {
+    ...props,
+    datasource: datasourceWithFullFolders,
+    controls: {
+      ...props.controls,
+      datasource: {
+        ...props.controls.datasource,
+        datasource: datasourceWithFullFolders,
+      },
+    },
+  };
+  render(
+    <ExploreContainer>
+      <DatasourcePanel {...propsWithFullFolders} />
+      <DndMetricSelect {...metricProps} />
+    </ExploreContainer>,
+    {
+      useRedux: true,
+      useDnd: true,
+    },
+  );
+
+  expect(screen.getByText('Test folder')).toBeInTheDocument();
+  expect(screen.getByText('Test nested folder')).toBeInTheDocument();
+  expect(screen.getByText('Second test folder')).toBeInTheDocument();
+  expect(screen.queryByText('Metrics')).not.toBeInTheDocument();
+  expect(screen.queryByText('Columns')).not.toBeInTheDocument();
+
+  columns.forEach(col => {
+    expect(screen.getByText(col.column_name)).toBeInTheDocument();
+  });
+
+  metrics.forEach(metric => {
+    expect(screen.getByText(metric.metric_name)).toBeInTheDocument();
+  });
+
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
+  expect(screen.getAllByTestId('datasource-panel-divider').length).toEqual(1);
+});

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelDragOption/DatasourcePanelDragOption.test.tsx

@@ -23,7 +23,7 @@ import DatasourcePanelDragOption from '.';
 test('should render', async () => {
   render(
     <DatasourcePanelDragOption
-      value={{ metric_name: 'test' }}
+      value={{ metric_name: 'test', uuid: '1' }}
       type={DndItemType.Metric}
     />,
     { useDnd: true },
@@ -38,7 +38,7 @@ test('should render', async () => {
 test('should have attribute draggable:true', async () => {
   render(
     <DatasourcePanelDragOption
-      value={{ metric_name: 'test' }}
+      value={{ metric_name: 'test', uuid: '1' }}
       type={DndItemType.Metric}
     />,
     { useDnd: true },

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelItem.test.tsx

@@ -20,178 +20,89 @@ import {
   columns,
   metrics,
 } from 'src/explore/components/DatasourcePanel/fixtures';
-import { fireEvent, render, within } from 'spec/helpers/testing-library';
-import DatasourcePanelItem from './DatasourcePanelItem';
+import { screen, userEvent, render } from 'spec/helpers/testing-library';
+import DatasourcePanelItem, {
+  DatasourcePanelItemProps,
+} from './DatasourcePanelItem';
 
-const mockData = {
-  metricSlice: metrics,
-  columnSlice: columns,
-  totalMetrics: Math.max(metrics.length, 10),
-  totalColumns: Math.max(columns.length, 13),
+const mockData: DatasourcePanelItemProps['data'] = {
+  flattenedItems: [
+    { type: 'header', depth: 0, folderId: '1', height: 50 },
+    ...metrics.map((m, idx) => ({
+      type: 'item' as const,
+      depth: 0,
+      folderId: '1',
+      height: 32,
+      index: idx,
+      item: { ...m, type: 'metric' as const },
+    })),
+    { type: 'divider', depth: 0, folderId: '1', height: 16 },
+    { type: 'header', depth: 0, folderId: '2', height: 50 },
+    ...columns.map((m, idx) => ({
+      type: 'item' as const,
+      depth: 0,
+      folderId: '2',
+      height: 32,
+      index: idx,
+      item: { ...m, type: 'column' as const },
+    })),
+  ],
+  folderMap: new Map([
+    [
+      '1',
+      {
+        id: '1',
+        isCollapsed: false,
+        name: 'Metrics',
+        items: metrics.map(m => ({ ...m, type: 'metric' })),
+        totalItems: metrics.length,
+        showingItems: metrics.length,
+      },
+    ],
+    [
+      '2',
+      {
+        id: '2',
+        isCollapsed: false,
+        name: 'Columns',
+        items: columns.map(c => ({ ...c, type: 'column' })),
+        totalItems: columns.length,
+        showingItems: columns.length,
+      },
+    ],
+  ]),
   width: 300,
-  showAllMetrics: false,
-  onShowAllMetricsChange: jest.fn(),
-  showAllColumns: false,
-  onShowAllColumnsChange: jest.fn(),
-  collapseMetrics: false,
-  onCollapseMetricsChange: jest.fn(),
-  collapseColumns: false,
-  onCollapseColumnsChange: jest.fn(),
-  hiddenMetricCount: 0,
-  hiddenColumnCount: 0,
+  onToggleCollapse: jest.fn(),
+  collapsedFolderIds: new Set(),
 };
 
-test('renders each item accordingly', () => {
-  const { getByText, getByTestId, rerender, container } = render(
-    <DatasourcePanelItem index={0} data={mockData} style={{}} />,
+const setup = (data: DatasourcePanelItemProps['data'] = mockData) =>
+  render(
+    <>
+      {data.flattenedItems.map((_, index) => (
+        <DatasourcePanelItem index={index} data={data} style={{}} />
+      ))}
+    </>,
     { useDnd: true },
   );
 
-  expect(getByText('Metrics')).toBeInTheDocument();
-  rerender(<DatasourcePanelItem index={1} data={mockData} style={{}} />);
-  expect(
-    getByText(
-      `Showing ${mockData.metricSlice.length} of ${mockData.totalMetrics}`,
-    ),
-  ).toBeInTheDocument();
-  mockData.metricSlice.forEach((metric, metricIndex) => {
-    rerender(
-      <DatasourcePanelItem
-        index={metricIndex + 2}
-        data={mockData}
-        style={{}}
-      />,
-    );
-    expect(getByTestId('DatasourcePanelDragOption')).toBeInTheDocument();
-    expect(
-      within(getByTestId('DatasourcePanelDragOption')).getByText(
-        metric.metric_name,
-      ),
-    ).toBeInTheDocument();
-  });
-  rerender(
-    <DatasourcePanelItem
-      index={2 + mockData.metricSlice.length}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  expect(container).toHaveTextContent('');
+test('renders each item accordingly', () => {
+  setup();
+  expect(screen.getByText('Metrics')).toBeInTheDocument();
+  expect(screen.getByText('metric_end_certified')).toBeInTheDocument();
+  expect(screen.getByText('metric_end')).toBeInTheDocument();
 
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  expect(getByText('Columns')).toBeInTheDocument();
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection + 1}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  expect(
-    getByText(
-      `Showing ${mockData.columnSlice.length} of ${mockData.totalColumns}`,
-    ),
-  ).toBeInTheDocument();
-  mockData.columnSlice.forEach((column, columnIndex) => {
-    rerender(
-      <DatasourcePanelItem
-        index={startIndexOfColumnSection + columnIndex + 2}
-        data={mockData}
-        style={{}}
-      />,
-    );
-    expect(getByTestId('DatasourcePanelDragOption')).toBeInTheDocument();
-    expect(
-      within(getByTestId('DatasourcePanelDragOption')).getByText(
-        column.column_name,
-      ),
-    ).toBeInTheDocument();
-  });
+  expect(screen.getByText('Columns')).toBeInTheDocument();
+  expect(screen.getByText('bootcamp_attend')).toBeInTheDocument();
+  expect(screen.getByText('calc_first_time_dev')).toBeInTheDocument();
+  expect(screen.getByText('aaaaaaaaaaa')).toBeInTheDocument();
+
+  expect(screen.getByTestId('datasource-panel-divider')).toBeInTheDocument();
+  expect(screen.getAllByTestId('DatasourcePanelDragOption').length).toEqual(5);
 });
 
 test('can collapse metrics and columns', () => {
-  mockData.onCollapseMetricsChange.mockClear();
-  mockData.onCollapseColumnsChange.mockClear();
-  const { queryByText, getByRole, rerender } = render(
-    <DatasourcePanelItem index={0} data={mockData} style={{}} />,
-    { useDnd: true },
-  );
-  fireEvent.click(getByRole('button'));
-  expect(mockData.onCollapseMetricsChange).toHaveBeenCalled();
-  expect(mockData.onCollapseColumnsChange).not.toHaveBeenCalled();
-
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection}
-      data={mockData}
-      style={{}}
-    />,
-  );
-  fireEvent.click(getByRole('button'));
-  expect(mockData.onCollapseColumnsChange).toHaveBeenCalled();
-
-  rerender(
-    <DatasourcePanelItem
-      index={1}
-      data={{
-        ...mockData,
-        collapseMetrics: true,
-      }}
-      style={{}}
-    />,
-  );
-  expect(
-    queryByText(
-      `Showing ${mockData.metricSlice.length} of ${mockData.totalMetrics}`,
-    ),
-  ).not.toBeInTheDocument();
-
-  rerender(
-    <DatasourcePanelItem
-      index={2}
-      data={{
-        ...mockData,
-        collapseMetrics: true,
-      }}
-      style={{}}
-    />,
-  );
-  expect(queryByText('Columns')).toBeInTheDocument();
-});
-
-test('shows ineligible items count', () => {
-  const hiddenColumnCount = 3;
-  const hiddenMetricCount = 1;
-  const dataWithHiddenItems = {
-    ...mockData,
-    hiddenColumnCount,
-    hiddenMetricCount,
-  };
-  const { getByText, rerender } = render(
-    <DatasourcePanelItem index={1} data={dataWithHiddenItems} style={{}} />,
-    { useDnd: true },
-  );
-  expect(
-    getByText(`${hiddenMetricCount} ineligible item(s) are hidden`),
-  ).toBeInTheDocument();
-
-  const startIndexOfColumnSection = mockData.metricSlice.length + 3;
-  rerender(
-    <DatasourcePanelItem
-      index={startIndexOfColumnSection + 1}
-      data={dataWithHiddenItems}
-      style={{}}
-    />,
-  );
-  expect(
-    getByText(`${hiddenColumnCount} ineligible item(s) are hidden`),
-  ).toBeInTheDocument();
+  setup();
+  userEvent.click(screen.getAllByRole('button')[0]);
+  expect(mockData.onToggleCollapse).toHaveBeenCalled();
 });

superset-frontend/src/explore/components/DatasourcePanel/DatasourcePanelItem.tsx

@@ -16,71 +16,31 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { CSSProperties, FC } from 'react';
+import { CSSProperties, ReactNode, useCallback } from 'react';
 
-import { css, Metric, styled, t, useTheme } from '@superset-ui/core';
+import {
+  css,
+  styled,
+  t,
+  useCSSTextTruncation,
+  useTheme,
+} from '@superset-ui/core';
 
 import { Icons } from 'src/components/Icons';
+import { Tooltip } from 'src/components/Tooltip';
 import DatasourcePanelDragOption from './DatasourcePanelDragOption';
 import { DndItemType } from '../DndItemType';
-import { DndItemValue } from './types';
-
-export type DataSourcePanelColumn = {
-  is_dttm?: boolean | null;
-  description?: string | null;
-  expression?: string | null;
-  is_certified?: number | null;
-  column_name?: string | null;
-  name?: string | null;
-  type?: string;
-};
-
-type Props = {
-  index: number;
-  style: CSSProperties;
-  data: {
-    metricSlice: Metric[];
-    columnSlice: DataSourcePanelColumn[];
-    totalMetrics: number;
-    totalColumns: number;
-    width: number;
-    showAllMetrics: boolean;
-    onShowAllMetricsChange: (showAll: boolean) => void;
-    showAllColumns: boolean;
-    onShowAllColumnsChange: (showAll: boolean) => void;
-    collapseMetrics: boolean;
-    onCollapseMetricsChange: (collapse: boolean) => void;
-    collapseColumns: boolean;
-    onCollapseColumnsChange: (collapse: boolean) => void;
-    hiddenMetricCount: number;
-    hiddenColumnCount: number;
-  };
-};
-
-export const DEFAULT_MAX_COLUMNS_LENGTH = 50;
-export const DEFAULT_MAX_METRICS_LENGTH = 50;
-export const ITEM_HEIGHT = 30;
-
-const Button = styled.button`
-  background: none;
-  border: none;
-  text-decoration: underline;
-  color: ${({ theme }) => theme.colors.primary.dark1};
-`;
-
-const ButtonContainer = styled.div`
-  text-align: center;
-  padding-top: 2px;
-`;
+import { DndItemValue, FlattenedItem, Folder } from './types';
 
 const LabelWrapper = styled.div`
   ${({ theme }) => css`
+    color: ${theme.colors.grayscale.dark1};
     overflow: hidden;
     text-overflow: ellipsis;
     font-size: ${theme.typography.sizes.s}px;
     background-color: ${theme.colors.grayscale.light4};
     margin: ${theme.gridUnit * 2}px 0;
-    border-radius: 4px;
+    border-radius: ${theme.borderRadius}px;
     padding: 0 ${theme.gridUnit}px;
 
     &:first-of-type {
@@ -117,98 +77,136 @@ const LabelWrapper = styled.div`
 `;
 
 const SectionHeaderButton = styled.button`
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
   border: none;
   background: transparent;
   width: 100%;
-  padding-inline: 0px;
+  height: 100%;
+  padding-inline: 0;
+`;
+
+const SectionHeaderTextContainer = styled.div`
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  width: 100%;
 `;
 
 const SectionHeader = styled.span`
-  ${({ theme }) => `
+  ${({ theme }) => css`
+    color: ${theme.colors.grayscale.dark1};
     font-size: ${theme.typography.sizes.m}px;
+    font-weight: ${theme.typography.weights.medium};
     line-height: 1.3;
-  `}
-`;
-
-const Box = styled.div`
-  ${({ theme }) => `
-    border: 1px ${theme.colors.grayscale.light4} solid;
-    border-radius: ${theme.gridUnit}px;
-    font-size: ${theme.typography.sizes.s}px;
-    padding: ${theme.gridUnit}px;
-    color: ${theme.colors.grayscale.light1};
-    text-overflow: ellipsis;
-    white-space: nowrap;
+    text-align: left;
+    display: -webkit-box;
+    -webkit-line-clamp: 1;
+    -webkit-box-orient: vertical;
     overflow: hidden;
+    text-overflow: ellipsis;
   `}
 `;
 
-const DatasourcePanelItem: FC<Props> = ({ index, style, data }) => {
-  const {
-    metricSlice: _metricSlice,
-    columnSlice,
-    totalMetrics,
-    totalColumns,
-    width,
-    showAllMetrics,
-    onShowAllMetricsChange,
-    showAllColumns,
-    onShowAllColumnsChange,
-    collapseMetrics,
-    onCollapseMetricsChange,
-    collapseColumns,
-    onCollapseColumnsChange,
-    hiddenMetricCount,
-    hiddenColumnCount,
-  } = data;
-  const metricSlice = collapseMetrics ? [] : _metricSlice;
+const Divider = styled.div`
+  ${({ theme }) => css`
+    height: 16px;
+    border-bottom: 1px solid ${theme.colors.grayscale.light3};
+  `}
+`;
 
-  const EXTRA_LINES = collapseMetrics ? 1 : 2;
-  const isColumnSection = collapseMetrics
-    ? index >= 1
-    : index > metricSlice.length + EXTRA_LINES;
-  const HEADER_LINE = isColumnSection
-    ? metricSlice.length + EXTRA_LINES + 1
-    : 0;
-  const SUBTITLE_LINE = HEADER_LINE + 1;
-  const BOTTOM_LINE =
-    (isColumnSection ? columnSlice.length : metricSlice.length) +
-    (collapseMetrics ? HEADER_LINE : SUBTITLE_LINE) +
-    1;
-  const collapsed = isColumnSection ? collapseColumns : collapseMetrics;
-  const setCollapse = isColumnSection
-    ? onCollapseColumnsChange
-    : onCollapseMetricsChange;
-  const showAll = isColumnSection ? showAllColumns : showAllMetrics;
-  const setShowAll = isColumnSection
-    ? onShowAllColumnsChange
-    : onShowAllMetricsChange;
+export interface DatasourcePanelItemProps {
+  index: number;
+  style: CSSProperties;
+  data: {
+    flattenedItems: FlattenedItem[];
+    folderMap: Map<string, Folder>;
+    width: number;
+    onToggleCollapse: (folderId: string) => void;
+    collapsedFolderIds: Set<string>;
+  };
+}
+
+const DatasourcePanelItem = ({
+  index,
+  style,
+  data,
+}: DatasourcePanelItemProps) => {
+  const {
+    flattenedItems,
+    folderMap,
+    width,
+    onToggleCollapse,
+    collapsedFolderIds,
+  } = data;
+  const item = flattenedItems[index];
   const theme = useTheme();
-  const hiddenCount = isColumnSection ? hiddenColumnCount : hiddenMetricCount;
+  const [labelRef, labelIsTruncated] = useCSSTextTruncation<HTMLSpanElement>({
+    isVertical: true,
+    isHorizontal: false,
+  });
+
+  const getTooltipNode = useCallback(
+    (folder: Folder) => {
+      let tooltipNode: ReactNode | null = null;
+      if (labelIsTruncated) {
+        tooltipNode = (
+          <div>
+            <b>{t('Name')}:</b> {folder.name}
+          </div>
+        );
+      }
+      if (folder.description) {
+        tooltipNode = (
+          <div>
+            {tooltipNode}
+            <div
+              css={
+                tooltipNode &&
+                css`
+                  margin-top: ${theme.gridUnit}px;
+                `
+              }
+            >
+              <b>{t('Description')}:</b> {folder.description}
+            </div>
+          </div>
+        );
+      }
+      return tooltipNode;
+    },
+    [labelIsTruncated],
+  );
+
+  if (!item) return null;
+
+  const folder = folderMap.get(item.folderId);
+  if (!folder) return null;
+
+  const indentation = item.depth * theme.gridUnit * 4;
 
   return (
     <div
-      style={style}
-      css={css`
-        padding: 0 ${theme.gridUnit * 4}px;
-      `}
+      style={{
+        ...style,
+        paddingLeft: theme.gridUnit * 4 + indentation,
+        paddingRight: theme.gridUnit * 4,
+      }}
     >
-      {index === HEADER_LINE && (
-        <SectionHeaderButton onClick={() => setCollapse(!collapsed)}>
-          <SectionHeader>
-            {isColumnSection ? t('Columns') : t('Metrics')}
-          </SectionHeader>
-          {collapsed ? (
-            <Icons.DownOutlined iconSize="s" />
-          ) : (
-            <Icons.UpOutlined iconSize="s" />
-          )}
+      {item.type === 'header' && (
+        <SectionHeaderButton onClick={() => onToggleCollapse(folder.id)}>
+          <Tooltip title={getTooltipNode(folder)}>
+            <SectionHeaderTextContainer>
+              <SectionHeader ref={labelRef}>{folder.name}</SectionHeader>
+              {collapsedFolderIds.has(folder.id) ? (
+                <Icons.DownOutlined iconSize="s" />
+              ) : (
+                <Icons.UpOutlined iconSize="s" />
+              )}
+            </SectionHeaderTextContainer>
+          </Tooltip>
         </SectionHeaderButton>
       )}
-      {index === SUBTITLE_LINE && !collapsed && (
+
+      {item.type === 'subtitle' && (
         <div
           css={css`
             display: flex;
@@ -223,46 +221,34 @@ const DatasourcePanelItem: FC<Props> = ({ index, style, data }) => {
               flex-shrink: 0;
             `}
           >
-            {isColumnSection
-              ? t(`Showing %s of %s`, columnSlice?.length, totalColumns)
-              : t(`Showing %s of %s`, metricSlice?.length, totalMetrics)}
+            {t(`Showing %s of %s items`, item.showingItems, item.totalItems)}
           </div>
-          {hiddenCount > 0 && (
-            <Box>{t(`%s ineligible item(s) are hidden`, hiddenCount)}</Box>
-          )}
         </div>
       )}
-      {index > SUBTITLE_LINE && index < BOTTOM_LINE && (
+
+      {item.type === 'item' && item.item && (
         <LabelWrapper
           key={
-            (isColumnSection
-              ? columnSlice[index - SUBTITLE_LINE - 1].column_name
-              : metricSlice[index - SUBTITLE_LINE - 1].metric_name) +
-            String(width)
+            (item.item.type === 'column'
+              ? item.item.column_name
+              : item.item.metric_name) + String(width)
           }
           className="column"
         >
           <DatasourcePanelDragOption
-            value={
-              isColumnSection
-                ? (columnSlice[index - SUBTITLE_LINE - 1] as DndItemValue)
-                : metricSlice[index - SUBTITLE_LINE - 1]
+            value={item.item as DndItemValue}
+            type={
+              item.item.type === 'column'
+                ? DndItemType.Column
+                : DndItemType.Metric
             }
-            type={isColumnSection ? DndItemType.Column : DndItemType.Metric}
           />
         </LabelWrapper>
       )}
-      {index === BOTTOM_LINE &&
-        !collapsed &&
-        (isColumnSection
-          ? totalColumns > DEFAULT_MAX_COLUMNS_LENGTH
-          : totalMetrics > DEFAULT_MAX_METRICS_LENGTH) && (
-          <ButtonContainer>
-            <Button onClick={() => setShowAll(!showAll)}>
-              {showAll ? t('Show less...') : t('Show all...')}
-            </Button>
-          </ButtonContainer>
-        )}
+
+      {item.type === 'divider' && (
+        <Divider data-test="datasource-panel-divider" />
+      )}
     </div>
   );
 };

superset-frontend/src/explore/components/DatasourcePanel/fixtures.tsx

@@ -26,6 +26,7 @@ export const columns = [
     filterable: true,
     groupby: true,
     id: 516,
+    uuid: '516',
     is_dttm: false,
     python_date_format: null,
     type: 'DOUBLE',
@@ -40,6 +41,7 @@ export const columns = [
     filterable: true,
     groupby: true,
     id: 477,
+    uuid: '477',
     is_dttm: false,
     python_date_format: null,
     type: 'VARCHAR',
@@ -52,7 +54,8 @@ export const columns = [
     expression: null,
     filterable: true,
     groupby: true,
-    id: 516,
+    id: 517,
+    uuid: '517',
     is_dttm: false,
     python_date_format: null,
     type: 'INT',
@@ -70,6 +73,7 @@ const metricsFiltered = {
       description: null,
       expression: '',
       id: 56,
+      uuid: '56',
       is_certified: true,
       metric_name: 'metric_end_certified',
       verbose_name: '',
@@ -84,6 +88,7 @@ const metricsFiltered = {
       description: null,
       expression: '',
       id: 57,
+      uuid: '57',
       is_certified: false,
       metric_name: 'metric_end',
       verbose_name: '',

superset-frontend/src/explore/components/DatasourcePanel/index.tsx

@@ -28,7 +28,6 @@ import {
 
 import { ControlConfig } from '@superset-ui/chart-controls';
 import AutoSizer from 'react-virtualized-auto-sizer';
-import { FixedSizeList as List } from 'react-window';
 
 import { matchSorter, rankings } from 'match-sorter';
 import Alert from 'src/components/Alert';
@@ -39,22 +38,19 @@ import { FAST_DEBOUNCE } from 'src/constants';
 import { ExploreActions } from 'src/explore/actions/exploreActions';
 import Control from 'src/explore/components/Control';
 import { useDebounceValue } from 'src/hooks/useDebounceValue';
-import DatasourcePanelItem, {
-  ITEM_HEIGHT,
-  DataSourcePanelColumn,
-  DEFAULT_MAX_COLUMNS_LENGTH,
-  DEFAULT_MAX_METRICS_LENGTH,
-} from './DatasourcePanelItem';
 import { DndItemType } from '../DndItemType';
-import { DndItemValue } from './types';
+import { DatasourceFolder, DatasourcePanelColumn, DndItemValue } from './types';
 import { DropzoneContext } from '../ExploreContainer';
+import { DatasourceItems } from './DatasourceItems';
+import { transformDatasourceWithFolders } from './transformDatasourceFolders';
 
 interface DatasourceControl extends Omit<ControlConfig, 'hidden'> {
   datasource?: IDatasource;
 }
 export interface IDatasource {
   metrics: Metric[];
-  columns: DataSourcePanelColumn[];
+  columns: DatasourcePanelColumn[];
+  folders?: DatasourceFolder[];
   id: number;
   type: DatasourceType;
   database: {
@@ -126,8 +122,18 @@ const StyledInfoboxWrapper = styled.div`
 
 const BORDER_WIDTH = 2;
 
-const sortCertifiedFirst = (slice: DataSourcePanelColumn[]) =>
-  slice.sort((a, b) => (b?.is_certified ?? 0) - (a?.is_certified ?? 0));
+const sortColumns = (slice: DatasourcePanelColumn[]) =>
+  [...slice]
+    .sort((col1, col2) => {
+      if (col1?.is_dttm && !col2?.is_dttm) {
+        return -1;
+      }
+      if (col2?.is_dttm && !col1?.is_dttm) {
+        return 1;
+      }
+      return 0;
+    })
+    .sort((a, b) => (b?.is_certified ?? 0) - (a?.is_certified ?? 0));
 
 export default function DataSourcePanel({
   datasource,
@@ -137,7 +143,7 @@ export default function DataSourcePanel({
   width,
 }: Props) {
   const [dropzones] = useContext(DropzoneContext);
-  const { columns: _columns, metrics } = datasource;
+  const { columns: _columns, metrics, folders: _folders } = datasource;
 
   const allowedColumns = useMemo(() => {
     const validators = Object.values(dropzones);
@@ -152,21 +158,6 @@ export default function DataSourcePanel({
     );
   }, [dropzones, _columns]);
 
-  // display temporal column first
-  const columns = useMemo(
-    () =>
-      [...allowedColumns].sort((col1, col2) => {
-        if (col1?.is_dttm && !col2?.is_dttm) {
-          return -1;
-        }
-        if (col2?.is_dttm && !col1?.is_dttm) {
-          return 1;
-        }
-        return 0;
-      }),
-    [allowedColumns],
-  );
-
   const allowedMetrics = useMemo(() => {
     const validators = Object.values(dropzones);
     return metrics.filter(metric =>
@@ -176,21 +167,15 @@ export default function DataSourcePanel({
     );
   }, [dropzones, metrics]);
 
-  const hiddenColumnCount = _columns.length - allowedColumns.length;
-  const hiddenMetricCount = metrics.length - allowedMetrics.length;
   const [showSaveDatasetModal, setShowSaveDatasetModal] = useState(false);
   const [inputValue, setInputValue] = useState('');
-  const [showAllMetrics, setShowAllMetrics] = useState(false);
-  const [showAllColumns, setShowAllColumns] = useState(false);
-  const [collapseMetrics, setCollapseMetrics] = useState(false);
-  const [collapseColumns, setCollapseColumns] = useState(false);
   const searchKeyword = useDebounceValue(inputValue, FAST_DEBOUNCE);
 
   const filteredColumns = useMemo(() => {
     if (!searchKeyword) {
-      return columns ?? [];
+      return allowedColumns ?? [];
     }
-    return matchSorter(columns, searchKeyword, {
+    return matchSorter(allowedColumns, searchKeyword, {
       keys: [
         {
           key: 'verbose_name',
@@ -211,7 +196,7 @@ export default function DataSourcePanel({
       ],
       keepDiacritics: true,
     });
-  }, [columns, searchKeyword]);
+  }, [allowedColumns, searchKeyword]);
 
   const filteredMetrics = useMemo(() => {
     if (!searchKeyword) {
@@ -244,22 +229,21 @@ export default function DataSourcePanel({
     });
   }, [allowedMetrics, searchKeyword]);
 
-  const metricSlice = useMemo(
-    () =>
-      showAllMetrics
-        ? filteredMetrics
-        : filteredMetrics?.slice?.(0, DEFAULT_MAX_METRICS_LENGTH),
-    [filteredMetrics, showAllMetrics],
+  const sortedColumns = useMemo(
+    () => sortColumns(filteredColumns),
+    [filteredColumns],
   );
 
-  const columnSlice = useMemo(
+  const folders = useMemo(
     () =>
-      showAllColumns
-        ? sortCertifiedFirst(filteredColumns)
-        : sortCertifiedFirst(
-            filteredColumns?.slice?.(0, DEFAULT_MAX_COLUMNS_LENGTH),
-          ),
-    [filteredColumns, showAllColumns],
+      transformDatasourceWithFolders(
+        filteredMetrics,
+        sortedColumns,
+        _folders,
+        allowedMetrics,
+        allowedColumns,
+      ),
+    [_folders, filteredMetrics, sortedColumns],
   );
 
   const showInfoboxCheck = () => {
@@ -324,57 +308,17 @@ export default function DataSourcePanel({
           )}
           <AutoSizer>
             {({ height }: { height: number }) => (
-              <List
+              <DatasourceItems
                 width={width - BORDER_WIDTH}
                 height={height}
-                itemSize={ITEM_HEIGHT}
-                itemCount={
-                  (collapseMetrics ? 0 : metricSlice?.length) +
-                  (collapseColumns ? 0 : columnSlice.length) +
-                  2 + // Each section header row
-                  (collapseMetrics ? 0 : 2) +
-                  (collapseColumns ? 0 : 2)
-                }
-                itemData={{
-                  metricSlice,
-                  columnSlice,
-                  width,
-                  totalMetrics: filteredMetrics.length,
-                  totalColumns: filteredColumns.length,
-                  showAllMetrics,
-                  onShowAllMetricsChange: setShowAllMetrics,
-                  showAllColumns,
-                  onShowAllColumnsChange: setShowAllColumns,
-                  collapseMetrics,
-                  onCollapseMetricsChange: setCollapseMetrics,
-                  collapseColumns,
-                  onCollapseColumnsChange: setCollapseColumns,
-                  hiddenMetricCount,
-                  hiddenColumnCount,
-                }}
-                overscanCount={5}
-              >
-                {DatasourcePanelItem}
-              </List>
+                folders={folders}
+              />
             )}
           </AutoSizer>
         </div>
       </>
     ),
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    [
-      columnSlice,
-      inputValue,
-      filteredColumns.length,
-      filteredMetrics.length,
-      metricSlice,
-      showAllColumns,
-      showAllMetrics,
-      collapseMetrics,
-      collapseColumns,
-      datasourceIsSaveable,
-      width,
-    ],
+    [inputValue, datasourceIsSaveable, width, folders],
   );
 
   return (

superset-frontend/src/explore/components/DatasourcePanel/transformDatasourceFolders.test.ts

@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Metric } from '@superset-ui/core';
+import { transformDatasourceWithFolders } from './transformDatasourceFolders';
+import { DatasourceFolder, DatasourcePanelColumn } from './types';
+
+const mockMetrics: Metric[] = [
+  { metric_name: 'metric1', uuid: 'metric1-uuid', expression: 'SUM(col1)' },
+  { metric_name: 'metric2', uuid: 'metric2-uuid', expression: 'AVG(col2)' },
+  { metric_name: 'metric3', uuid: 'metric3-uuid', expression: 'COUNT(*)' },
+];
+
+const mockColumns: DatasourcePanelColumn[] = [
+  { column_name: 'column1', uuid: 'column1-uuid', type: 'STRING' },
+  { column_name: 'column2', uuid: 'column2-uuid', type: 'NUMERIC' },
+  { column_name: 'column3', uuid: 'column3-uuid', type: 'DATETIME' },
+];
+
+test('transforms data into default folders when no folder config is provided', () => {
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    undefined,
+    mockMetrics,
+    mockColumns,
+  );
+
+  expect(result).toHaveLength(2);
+
+  expect(result[0].id).toBe('metrics-default');
+  expect(result[0].name).toBe('Metrics');
+  expect(result[0].items).toHaveLength(3);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+  expect(result[0].items[0].type).toBe('metric');
+
+  expect(result[1].id).toBe('columns-default');
+  expect(result[1].name).toBe('Columns');
+  expect(result[1].items).toHaveLength(3);
+  expect(result[1].items[0].uuid).toBe('column1-uuid');
+  expect(result[1].items[0].type).toBe('column');
+});
+
+test('transforms data according to folder configuration', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'folder1',
+      type: 'folder',
+      name: 'Important Metrics',
+      description: 'Key metrics folder',
+      children: [
+        { type: 'metric', uuid: 'metric1-uuid', name: 'metric1' },
+        { type: 'metric', uuid: 'metric2-uuid', name: 'metric2' },
+      ],
+    },
+    {
+      uuid: 'folder2',
+      type: 'folder',
+      name: 'Key Dimensions',
+      children: [{ type: 'column', uuid: 'column1-uuid', name: 'column1' }],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+    mockMetrics,
+    mockColumns,
+  );
+
+  // We expect 4 folders:
+  // 1. Important Metrics (from config)
+  // 2. Key Dimensions (from config)
+  // 3. Metrics (default for unassigned metrics)
+  // 4. Columns (default for unassigned columns)
+  expect(result).toHaveLength(4);
+
+  expect(result[0].id).toBe('folder1');
+  expect(result[0].name).toBe('Important Metrics');
+  expect(result[0].description).toBe('Key metrics folder');
+  expect(result[0].items).toHaveLength(2);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+
+  expect(result[1].id).toBe('folder2');
+  expect(result[1].name).toBe('Key Dimensions');
+  expect(result[1].items).toHaveLength(1);
+  expect(result[1].items[0].uuid).toBe('column1-uuid');
+
+  expect(result[2].id).toBe('metrics-default');
+  expect(result[2].items).toHaveLength(1);
+  expect(result[2].items[0].uuid).toBe('metric3-uuid');
+
+  expect(result[3].id).toBe('columns-default');
+  expect(result[3].items).toHaveLength(2);
+});
+
+test('handles nested folder structures', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'parent-folder',
+      type: 'folder',
+      name: 'Parent Folder',
+      children: [
+        {
+          uuid: 'child-folder',
+          type: 'folder',
+          name: 'Child Folder',
+          children: [{ type: 'metric', uuid: 'metric1-uuid', name: 'metric1' }],
+        },
+        { type: 'column', uuid: 'column1-uuid', name: 'column1' },
+      ],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+    mockMetrics,
+    mockColumns,
+  );
+
+  expect(result[0].id).toBe('parent-folder');
+  expect(result[0].name).toBe('Parent Folder');
+  expect(result[0].items).toHaveLength(1);
+  expect(result[0].subFolders).toHaveLength(1);
+
+  const childFolder = result[0].subFolders![0];
+  expect(childFolder.id).toBe('child-folder');
+  expect(childFolder.name).toBe('Child Folder');
+  expect(childFolder.items).toHaveLength(1);
+  expect(childFolder.parentId).toBe('parent-folder');
+});
+
+test('handles empty children arrays', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'empty-folder',
+      type: 'folder',
+      name: 'Empty Folder',
+      children: [],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+    mockMetrics,
+    mockColumns,
+  );
+
+  expect(result[0].id).toBe('empty-folder');
+  expect(result[0].name).toBe('Empty Folder');
+  expect(result[0].items).toHaveLength(0);
+});
+
+test('handles non-existent metric and column UUIDs in folder config', () => {
+  const folderConfig: DatasourceFolder[] = [
+    {
+      uuid: 'folder1',
+      type: 'folder',
+      name: 'Test Folder',
+      children: [
+        {
+          type: 'metric',
+          uuid: 'non-existent-metric',
+          name: 'Missing Metric',
+        },
+        {
+          type: 'column',
+          uuid: 'non-existent-column',
+          name: 'Missing Column',
+        },
+        { type: 'metric', uuid: 'metric1-uuid', name: 'Existing Metric' },
+      ],
+    },
+  ];
+
+  const result = transformDatasourceWithFolders(
+    mockMetrics,
+    mockColumns,
+    folderConfig,
+    mockMetrics,
+    mockColumns,
+  );
+
+  expect(result[0].id).toBe('folder1');
+  expect(result[0].items).toHaveLength(1);
+  expect(result[0].items[0].uuid).toBe('metric1-uuid');
+});

superset-frontend/src/explore/components/DatasourcePanel/transformDatasourceFolders.ts

@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { Metric, t } from '@superset-ui/core';
+import {
+  ColumnItem,
+  DatasourceFolder,
+  DatasourcePanelColumn,
+  Folder,
+  MetricItem,
+} from './types';
+
+const transformToFolderStructure = (
+  metricsToDisplay: MetricItem[],
+  columnsToDisplay: ColumnItem[],
+  folderConfig: DatasourceFolder[] | undefined,
+  allMetrics: Metric[],
+  allColumns: DatasourcePanelColumn[],
+): Folder[] => {
+  const metricsMap = new Map<string, MetricItem>();
+  const columnsMap = new Map<string, ColumnItem>();
+
+  metricsToDisplay.forEach(metric => {
+    metricsMap.set(metric.uuid, metric);
+  });
+
+  columnsToDisplay.forEach(column => {
+    columnsMap.set(column.uuid, column);
+  });
+
+  let metricsInFolders = 0;
+  let columnsInFolders = 0;
+  const processFolder = (
+    datasourceFolder: DatasourceFolder,
+    parentId?: string,
+  ): Folder => {
+    const folder: Folder = {
+      id: datasourceFolder.uuid,
+      name: datasourceFolder.name,
+      description: datasourceFolder.description,
+      isCollapsed: false,
+      items: [],
+      totalItems: 0,
+      showingItems: 0,
+      parentId,
+    };
+
+    if (datasourceFolder.children && datasourceFolder.children.length > 0) {
+      if (!folder.subFolders) {
+        folder.subFolders = [];
+      }
+
+      datasourceFolder.children.forEach(child => {
+        if (child.type === 'folder') {
+          const subFolder = processFolder(child as DatasourceFolder, folder.id);
+          folder.subFolders!.push(subFolder);
+          folder.totalItems += subFolder.totalItems;
+          folder.showingItems += subFolder.showingItems;
+        } else if (child.type === 'metric') {
+          folder.totalItems += 1;
+          metricsInFolders += 1;
+          const metric = metricsMap.get(child.uuid);
+          if (metric) {
+            folder.items.push(metric);
+            metricsMap.delete(metric.uuid);
+            folder.showingItems += 1;
+          }
+        } else if (child.type === 'column') {
+          folder.totalItems += 1;
+          columnsInFolders += 1;
+          const column = columnsMap.get(child.uuid);
+          if (column) {
+            folder.items.push(column);
+            columnsMap.delete(column.uuid);
+            folder.showingItems += 1;
+          }
+        }
+      });
+    }
+
+    return folder;
+  };
+
+  if (!folderConfig) {
+    return [
+      {
+        id: 'metrics-default',
+        name: t('Metrics'),
+        isCollapsed: false,
+        items: metricsToDisplay,
+        totalItems: allMetrics.length,
+        showingItems: metricsToDisplay.length,
+      },
+      {
+        id: 'columns-default',
+        name: t('Columns'),
+        isCollapsed: false,
+        items: columnsToDisplay,
+        totalItems: allColumns.length,
+        showingItems: columnsToDisplay.length,
+      },
+    ];
+  }
+
+  const folders = folderConfig.map(config => processFolder(config));
+
+  const unassignedMetrics = metricsToDisplay.filter(metric =>
+    metricsMap.has(metric.uuid),
+  );
+  const unassignedColumns = columnsToDisplay.filter(column =>
+    columnsMap.has(column.uuid),
+  );
+
+  if (unassignedMetrics.length > 0) {
+    folders.push({
+      id: 'metrics-default',
+      name: t('Metrics'),
+      isCollapsed: false,
+      items: unassignedMetrics,
+      totalItems: allMetrics.length - metricsInFolders,
+      showingItems: unassignedMetrics.length,
+    });
+  }
+
+  if (unassignedColumns.length > 0) {
+    folders.push({
+      id: 'columns-default',
+      name: t('Columns'),
+      isCollapsed: false,
+      items: unassignedColumns,
+      totalItems: allColumns.length - columnsInFolders,
+      showingItems: unassignedColumns.length,
+    });
+  }
+
+  return folders;
+};
+
+export const transformDatasourceWithFolders = (
+  metricsToDisplay: Metric[],
+  columnsToDisplay: DatasourcePanelColumn[],
+  folderConfig: DatasourceFolder[] | undefined,
+  allMetrics: Metric[],
+  allColumns: DatasourcePanelColumn[],
+): Folder[] => {
+  const metricsWithType: MetricItem[] = metricsToDisplay.map(metric => ({
+    ...metric,
+    type: 'metric',
+  }));
+  const columnsWithType: ColumnItem[] = columnsToDisplay.map(column => ({
+    ...column,
+    type: 'column',
+  }));
+
+  return transformToFolderStructure(
+    metricsWithType,
+    columnsWithType,
+    folderConfig,
+    allMetrics,
+    allColumns,
+  );
+};

superset-frontend/src/explore/components/DatasourcePanel/types.ts

@@ -35,3 +35,59 @@ export function isDatasourcePanelDndItem(
 export function isSavedMetric(item: any): item is Metric {
   return item?.metric_name;
 }
+
+export type DatasourcePanelColumn = {
+  uuid: string;
+  id?: number;
+  is_dttm?: boolean | null;
+  description?: string | null;
+  expression?: string | null;
+  is_certified?: number | null;
+  column_name?: string | null;
+  name?: string | null;
+  type?: string;
+};
+
+export type DatasourceFolder = {
+  uuid: string;
+  type: 'folder';
+  name: string;
+  description?: string;
+  children?: (
+    | DatasourceFolder
+    | { type: 'metric'; uuid: string; name: string }
+    | { type: 'column'; uuid: string; name: string }
+  )[];
+};
+
+export type MetricItem = Metric & {
+  type: 'metric';
+};
+
+export type ColumnItem = DatasourcePanelColumn & {
+  type: 'column';
+};
+
+export type FolderItem = MetricItem | ColumnItem;
+
+export interface Folder {
+  id: string;
+  name: string;
+  description?: string;
+  isCollapsed: boolean;
+  items: FolderItem[];
+  subFolders?: Folder[];
+  parentId?: string;
+  totalItems: number;
+  showingItems: number; // items shown after filtering
+}
+
+export interface FlattenedItem {
+  type: 'header' | 'item' | 'divider' | 'subtitle';
+  folderId: string;
+  depth: number;
+  item?: FolderItem;
+  height: number;
+  totalItems?: number;
+  showingItems?: number;
+}