From 5b8e94de2671ff4e5048571958fae5fc1eaea412 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 10 Jul 2026 15:18:51 -0400 Subject: [PATCH] feat(semantic layers): dashboard filters (#40475) --- .../superset_core/semantic_layers/types.py | 2 +- .../src/query/types/Dashboard.ts | 2 + .../FilterBar/FilterControls/FilterValue.tsx | 9 +- .../FiltersConfigForm/ColumnSelect.tsx | 92 +++++--- .../FiltersConfigForm/DatasetSelect.test.tsx | 45 ++++ .../FiltersConfigForm/DatasetSelect.tsx | 102 ++++++++- .../FiltersConfigForm/FiltersConfigForm.tsx | 196 +++++++++++++----- .../FiltersConfigForm/getControlItemsMap.tsx | 4 + .../FiltersConfigForm/utils.test.ts | 76 ++++++- .../FiltersConfigForm/utils.ts | 91 +++++++- .../transformers/buildTarget.test.ts | 74 +++++++ .../transformers/buildTarget.ts | 61 ++++++ .../transformers/customizationTransformer.ts | 13 +- .../transformers/filterTransformer.ts | 16 +- .../nativeFilters/FiltersConfigModal/types.ts | 3 + .../nativeFilters/FiltersConfigModal/utils.ts | 10 +- .../components/nativeFilters/utils.ts | 6 +- .../dashboard/reducers/dashboardInfo.test.ts | 24 ++- .../src/dashboard/reducers/dashboardInfo.ts | 1 + superset-frontend/src/dashboard/types.ts | 3 + .../charts/getFormDataWithExtraFilters.ts | 17 +- .../util/getFormDataWithExtraFilters.test.ts | 43 ++++ .../dashboard/util/getRelatedCharts.test.ts | 65 +++++- .../src/dashboard/util/getRelatedCharts.ts | 15 +- .../datasets/DatasetSelectLabel/index.tsx | 24 ++- .../src/features/semanticLayers/label.ts | 12 ++ superset/semantic_layers/api.py | 9 +- superset/semantic_layers/mapper.py | 15 +- tests/unit_tests/semantic_layers/api_test.py | 3 + .../unit_tests/semantic_layers/mapper_test.py | 22 ++ 30 files changed, 917 insertions(+), 138 deletions(-) create mode 100644 superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/transformers/buildTarget.test.ts create mode 100644 superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/transformers/buildTarget.ts diff --git a/superset-core/src/superset_core/semantic_layers/types.py b/superset-core/src/superset_core/semantic_layers/types.py index 3bfa9e8c315..c26667b8bb2 100644 --- a/superset-core/src/superset_core/semantic_layers/types.py +++ b/superset-core/src/superset_core/semantic_layers/types.py @@ -158,7 +158,7 @@ class Filter: type: PredicateType column: Dimension | Metric | None operator: Operator - value: FilterValues | frozenset[FilterValues] + value: FilterValues | tuple[FilterValues, ...] | frozenset[FilterValues] class OrderDirection(enum.Enum): diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Dashboard.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Dashboard.ts index 40cdbbcad3d..51d8f4b12a4 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/Dashboard.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/Dashboard.ts @@ -18,6 +18,7 @@ */ import { AdhocFilter, DataMask } from '@superset-ui/core'; +import { DatasourceType } from './Datasource'; export interface ColumnOption { label: string; @@ -38,6 +39,7 @@ export interface NativeFilterScope { export interface NativeFilterTarget { datasetId: number; column: NativeFilterColumn; + datasourceType?: DatasourceType; // maybe someday support this? // show values from these columns in the filter options selector diff --git a/superset-frontend/src/dashboard/components/nativeFilters/FilterBar/FilterControls/FilterValue.tsx b/superset-frontend/src/dashboard/components/nativeFilters/FilterBar/FilterControls/FilterValue.tsx index 8f684f4a6c0..ef01b8cfcfc 100644 --- a/superset-frontend/src/dashboard/components/nativeFilters/FilterBar/FilterControls/FilterValue.tsx +++ b/superset-frontend/src/dashboard/components/nativeFilters/FilterBar/FilterControls/FilterValue.tsx @@ -31,6 +31,7 @@ import { ChartDataResponseResult, Behavior, DataMask, + DatasourceType, isFeatureEnabled, FeatureFlag, getChartMetadataRegistry, @@ -147,8 +148,13 @@ const FilterValue: FC = ({ const [target] = targets || []; const { datasetId, + datasourceType, column = {}, - }: Partial<{ datasetId: number; column: { name?: string } }> = target || {}; + }: Partial<{ + datasetId: number; + datasourceType: DatasourceType; + column: { name?: string }; + }> = target || {}; const groupby = column?.name; const hasDataSource = !!datasetId; const [isLoading, setIsLoading] = useState(hasDataSource); @@ -182,6 +188,7 @@ const FilterValue: FC = ({ const newFormData = getFormData({ ...filter, datasetId, + datasourceType, dependencies, groupby, adhoc_filters: adhocFilters, diff --git a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/ColumnSelect.tsx b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/ColumnSelect.tsx index 2e8f43bb21f..aa4761dd90a 100644 --- a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/ColumnSelect.tsx +++ b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/ColumnSelect.tsx @@ -19,9 +19,12 @@ import { useCallback, useState, useMemo, useEffect } from 'react'; import rison from 'rison'; import { t } from '@apache-superset/core/translation'; +import { GenericDataType } from '@apache-superset/core/common'; import { Column, + DatasourceType, ensureIsArray, + JsonResponse, useChangeEffect, getClientErrorObject, } from '@superset-ui/core'; @@ -29,6 +32,7 @@ import { type FormInstance, Select } from '@superset-ui/core/components'; import { useToasts } from 'src/components/MessageToasts/withToasts'; import { cachedSupersetGet } from 'src/utils/cachedSupersetGet'; import { NativeFiltersForm, NativeFiltersFormItem } from '../types'; +import { mapSemanticTypeToGenericDataType } from './utils'; interface ColumnSelectProps { allowClear?: boolean; @@ -37,6 +41,7 @@ interface ColumnSelectProps { formField?: keyof NativeFiltersFormItem; filterId: string; datasetId?: number; + datasourceType?: DatasourceType; value?: string | string[]; onChange?: (value: string) => void; mode?: 'multiple'; @@ -51,6 +56,7 @@ export function ColumnSelect({ formField = 'column', filterId, datasetId, + datasourceType, value, onChange, mode, @@ -86,25 +92,69 @@ export function ColumnSelect({ } }, [currentColumn, currentFilterType, resetColumnField]); - useChangeEffect(datasetId, previous => { + // Use a compound key so the effect re-fires when either the dataset ID or + // the datasource type changes. Datasets and semantic views have independent + // ID sequences, so switching between them with the same numeric ID must still + // trigger a column re-fetch. + const datasourceKey = `${datasetId}__${datasourceType || DatasourceType.Table}`; + useChangeEffect(datasourceKey, previous => { if (previous != null) { setColumns([]); resetColumnField(); } if (datasetId != null) { setLoading(true); - cachedSupersetGet({ - endpoint: `/api/v1/dataset/${datasetId}?q=${rison.encode({ - columns: [ - 'columns.column_name', - 'columns.is_dttm', - 'columns.type_generic', - 'columns.filterable', - ], - })}`, - }) - .then( - ({ json: { result } }) => { + const handleError = async ( + badResponse: Parameters[0], + ) => { + const { error, message } = await getClientErrorObject(badResponse); + let errorText = message || error || t('An error has occurred'); + if (message === 'Forbidden') { + errorText = t('You do not have permission to edit this dashboard'); + } + addDangerToast(errorText); + }; + + if (datasourceType === DatasourceType.SemanticView) { + cachedSupersetGet({ + endpoint: `/api/v1/semantic_view/${datasetId}/structure`, + }) + .then((response: JsonResponse) => { + const { dimensions = [] } = response.json?.result ?? {}; + const cols: Column[] = dimensions.map( + (dim: { name: string; type: string }) => { + const mappedType = mapSemanticTypeToGenericDataType(dim.type); + return { + column_name: dim.name, + type: dim.type, + is_dttm: mappedType === GenericDataType.Temporal, + type_generic: mappedType, + filterable: true, + }; + }, + ); + const lookupValue = Array.isArray(value) ? value : [value]; + const valueExists = cols.some((column: Column) => + lookupValue?.includes(column.column_name), + ); + if (!valueExists) { + resetColumnField(); + } + setColumns(cols); + }, handleError) + .finally(() => setLoading(false)); + } else { + cachedSupersetGet({ + endpoint: `/api/v1/dataset/${datasetId}?q=${rison.encode({ + columns: [ + 'columns.column_name', + 'columns.is_dttm', + 'columns.type_generic', + 'columns.filterable', + ], + })}`, + }) + .then(({ json: { result } }) => { const lookupValue = Array.isArray(value) ? value : [value]; const valueExists = result.columns.some((column: Column) => lookupValue?.includes(column.column_name), @@ -113,19 +163,9 @@ export function ColumnSelect({ resetColumnField(); } setColumns(result.columns); - }, - async badResponse => { - const { error, message } = await getClientErrorObject(badResponse); - let errorText = message || error || t('An error has occurred'); - if (message === 'Forbidden') { - errorText = t( - 'You do not have permission to edit this dashboard', - ); - } - addDangerToast(errorText); - }, - ) - .finally(() => setLoading(false)); + }, handleError) + .finally(() => setLoading(false)); + } } }); diff --git a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.test.tsx b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.test.tsx index fb8632b09d5..be579c9848b 100644 --- a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.test.tsx +++ b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.test.tsx @@ -265,3 +265,48 @@ test('returns total count from API when data is filtered', async () => { expect(result.data).toHaveLength(2); expect(result.data.find(item => item.value === 2)).toBeUndefined(); }); + +test('does not exclude semantic views that share dataset IDs', async () => { + supersetGetCache.clear(); + fetchMock.clearHistory().removeRoutes(); + + const originalFeatureFlags = window.featureFlags; + window.featureFlags = { + ...originalFeatureFlags, + SEMANTIC_LAYERS: true, + }; + + try { + fetchMock.get('glob:*/api/v1/datasource/*', { + result: [ + { + id: 7, + table_name: 'orders_dataset', + kind: 'physical', + database: { database_name: 'examples' }, + schema: 'public', + }, + { + id: 7, + table_name: 'orders_semantic_view', + kind: 'semantic_view', + database: { database_name: 'semantic_layer' }, + schema: null, + }, + ], + count: 2, + }); + + const result = await loadDatasetOptions('', 0, 100, [7]); + + expect(result.totalCount).toBe(2); + expect(result.data).toHaveLength(1); + expect(result.data[0]).toMatchObject({ + value: 'sv:7', + kind: 'semantic_view', + table_name: 'orders_semantic_view', + }); + } finally { + window.featureFlags = originalFeatureFlags; + } +}); diff --git a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.tsx b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.tsx index d089e98a675..71285fff831 100644 --- a/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.tsx +++ b/superset-frontend/src/dashboard/components/nativeFilters/FiltersConfigModal/FiltersConfigForm/DatasetSelect.tsx @@ -20,6 +20,8 @@ import { useCallback, useMemo, ReactNode } from 'react'; import rison from 'rison'; import { t } from '@apache-superset/core/translation'; import { + isFeatureEnabled, + FeatureFlag, JsonResponse, ClientErrorObject, getClientErrorObject, @@ -37,8 +39,12 @@ import { } from 'src/features/semanticLayers/label'; interface DatasetSelectProps { - onChange: (value: { label: string | ReactNode; value: number }) => void; - value?: { label: string | ReactNode; value: number }; + onChange: (value: { + label: string | ReactNode; + value: number; + kind?: string; + }) => void; + value?: { label: string | ReactNode; value: number; kind?: string }; excludeDatasetIds?: number[]; } @@ -50,37 +56,81 @@ const getErrorMessage = ({ error, message }: ClientErrorObject) => { return errorText; }; +/** + * Builds a unique select-option value for the combined datasource endpoint. + * Datasets and semantic views have independent integer ID sequences, so we + * prefix with a type tag to avoid collisions in AsyncSelect's dedup logic. + */ +const toCompositeValue = (id: number, kind?: string): string => + kind === 'semantic_view' ? `sv:${id}` : `ds:${id}`; + +/** Extracts the numeric ID from a composite "sv:123" / "ds:456" string. */ +const fromCompositeValue = (compositeValue: string | number): number => { + if (typeof compositeValue !== 'string') return compositeValue; + const parts = compositeValue.split(':'); + return parts.length === 2 + ? parseInt(parts[1], 10) + : parseInt(compositeValue, 10); +}; + +/** Derives the `kind` value from a composite string prefix. */ +const kindFromComposite = (compositeValue: string): string | undefined => + compositeValue.startsWith('sv:') ? 'semantic_view' : undefined; + +const isExcludedDatasource = ( + item: Dataset, + excludeDatasetIds: number[], +): boolean => { + if (!excludeDatasetIds.includes(item.id)) { + return false; + } + + return item.kind !== 'semantic_view'; +}; + export const loadDatasetOptions = async ( search: string, page: number, pageSize: number, excludeDatasetIds: number[] = [], ) => { + const useSemanticLayers = isFeatureEnabled(FeatureFlag.SemanticLayers); const query = rison.encode({ - columns: ['id', 'table_name', 'database.database_name', 'schema'], + ...(useSemanticLayers + ? {} + : { + columns: ['id', 'table_name', 'database.database_name', 'schema'], + }), filters: [{ col: 'table_name', opr: 'ct', value: search }], page, page_size: pageSize, order_column: 'table_name', order_direction: 'asc', }); + const endpoint = useSemanticLayers + ? `/api/v1/datasource/?q=${query}` + : `/api/v1/dataset/?q=${query}`; return cachedSupersetGet({ - endpoint: `/api/v1/dataset/?q=${query}`, + endpoint, }) .then((response: JsonResponse) => { const filteredResult = response.json.result.filter( - (item: Dataset) => !excludeDatasetIds.includes(item.id), + (item: Dataset) => !isExcludedDatasource(item, excludeDatasetIds), ); const list: { label: string | ReactNode; value: string | number; table_name: string; + kind?: string; }[] = filteredResult.map((item: Dataset) => ({ ...item, label: DatasetSelectLabel(item), - value: item.id, + value: useSemanticLayers + ? toCompositeValue(item.id, item.kind) + : item.id, table_name: item.table_name, + kind: item.kind, })); return { data: list, @@ -98,18 +148,54 @@ const DatasetSelect = ({ value, excludeDatasetIds = [], }: DatasetSelectProps) => { + const useSemanticLayers = isFeatureEnabled(FeatureFlag.SemanticLayers); + const loadDatasetOptionsCallback = useCallback( (search: string, page: number, pageSize: number) => loadDatasetOptions(search, page, pageSize, excludeDatasetIds), [excludeDatasetIds], ); + // Convert the external numeric value to the composite string format that + // AsyncSelect needs for matching against the loaded options. + const selectValue = useMemo(() => { + if (!value || !useSemanticLayers) return value; + return { + ...value, + value: toCompositeValue(value.value, value.kind), + }; + }, [value, useSemanticLayers]); + + // Convert the composite string value from the selected option back to a + // numeric ID before passing it to the external onChange handler. + // AsyncSelect's first argument is a LabeledValue ({key, label, value}) and + // does NOT include custom option fields like `kind`. We derive `kind` from + // the composite value prefix so consumers can distinguish datasource types. + const handleChange = useCallback( + (selected: { + label: string | ReactNode; + value: number | string; + kind?: string; + }) => { + if (typeof selected.value === 'string') { + onChange({ + ...selected, + value: fromCompositeValue(selected.value), + kind: kindFromComposite(selected.value), + }); + } else { + onChange(selected as Parameters[0]); + } + }, + [onChange], + ); + return ( { + if (formFilter?.datasourceType) { + return formFilter.datasourceType; + } + if (isChartCustomization) { + return ( + customizationToEdit?.targets?.[0]?.datasourceType || + DatasourceType.Table + ); + } + return filterToEdit?.targets?.[0]?.datasourceType || DatasourceType.Table; + }; + + const datasourceType = getDatasourceType(); + const formChanged = useCallback(() => { form.setFields([ { @@ -425,6 +443,7 @@ const FiltersConfigForm = ( ? getControlItemsMap({ expanded, datasetId, + datasourceType, disabled: false, forceUpdate, formChanged, @@ -496,6 +515,7 @@ const FiltersConfigForm = ( } const formData = getFormData({ datasetId, + datasourceType, dashboardId, groupby: formFilter?.column, ...formFilter, @@ -566,6 +586,7 @@ const FiltersConfigForm = ( const newFormData = getFormData({ datasetId, + datasourceType, groupby: hasColumn ? formFilter?.column : undefined, ...formFilter, }); @@ -741,45 +762,93 @@ const FiltersConfigForm = ( useEffect(() => { if (datasetId) { - cachedSupersetGet({ - endpoint: `/api/v1/dataset/${datasetId}?q=${rison.encode({ - columns: [ - 'columns.column_name', - 'columns.expression', - 'columns.filterable', - 'columns.is_dttm', - 'columns.type', - 'columns.type_generic', - 'columns.verbose_name', - 'database.id', - 'database.database_name', - 'datasource_type', - 'filter_select_enabled', - 'id', - 'is_sqllab_view', - 'main_dttm_col', - 'metrics.metric_name', - 'metrics.verbose_name', - 'schema', - 'sql', - 'table_name', - 'time_grain_sqla', - ], - })}`, - }) - .then((response: JsonResponse) => { - setMetrics(response.json?.result?.metrics); - const dataset = response.json?.result; - // modify the response to fit structure expected by AdhocFilterControl - dataset.type = dataset.datasource_type; - dataset.filter_select = true; - setDatasetDetails(dataset); + if (datasourceType === DatasourceType.SemanticView) { + cachedSupersetGet({ + endpoint: `/api/v1/semantic_view/${datasetId}/structure`, }) - .catch((response: SupersetApiError) => { - addDangerToast(response.message); - }); + .then((response: JsonResponse) => { + const { + name: svName, + dimensions = [], + metrics: svMetrics = [], + } = response.json?.result ?? {}; + const columns = dimensions.map( + (dim: { name: string; type: string }) => { + const mappedType = mapSemanticTypeToGenericDataType(dim.type); + return { + column_name: dim.name, + type: dim.type, + is_dttm: mappedType === GenericDataType.Temporal, + filterable: true, + type_generic: mappedType, + }; + }, + ); + const mappedMetrics = svMetrics.map( + (m: { name: string; definition: string }) => ({ + metric_name: m.name, + expression: m.definition, + verbose_name: null, + }), + ); + setMetrics(mappedMetrics); + setDatasetDetails({ + columns, + metrics: mappedMetrics, + datasource_type: DatasourceType.SemanticView, + type: DatasourceType.SemanticView, + filter_select: true, + filter_select_enabled: true, + time_grain_sqla: [], + main_dttm_col: null, + id: datasetId, + table_name: svName, + }); + }) + .catch((response: SupersetApiError) => { + addDangerToast(response.message); + }); + } else { + cachedSupersetGet({ + endpoint: `/api/v1/dataset/${datasetId}?q=${rison.encode({ + columns: [ + 'columns.column_name', + 'columns.expression', + 'columns.filterable', + 'columns.is_dttm', + 'columns.type', + 'columns.type_generic', + 'columns.verbose_name', + 'database.id', + 'database.database_name', + 'datasource_type', + 'filter_select_enabled', + 'id', + 'is_sqllab_view', + 'main_dttm_col', + 'metrics.metric_name', + 'metrics.verbose_name', + 'schema', + 'sql', + 'table_name', + 'time_grain_sqla', + ], + })}`, + }) + .then((response: JsonResponse) => { + setMetrics(response.json?.result?.metrics); + const dataset = response.json?.result; + // modify the response to fit structure expected by AdhocFilterControl + dataset.type = dataset.datasource_type; + dataset.filter_select = true; + setDatasetDetails(dataset); + }) + .catch((response: SupersetApiError) => { + addDangerToast(response.message); + }); + } } - }, [datasetId]); + }, [datasetId, datasourceType]); useImperativeHandle(ref, () => ({ changeTab(tab: 'configuration' | 'scoping') { @@ -818,7 +887,15 @@ const FiltersConfigForm = ( if (chartDatasetUid === undefined) { return; } - if (loadedDatasets[chartDatasetUid]?.id !== formFilter?.dataset?.value) { + + const matchesFilterDatasource = doesChartMatchFilterDatasource( + chartDatasetUid, + loadedDatasets, + formFilter.dataset.value, + datasourceType, + ); + + if (!matchesFilterDatasource) { excluded.push(chart.id); } }); @@ -826,6 +903,7 @@ const FiltersConfigForm = ( }, [ JSON.stringify(Object.values(charts).map(chart => chart.id)), formFilter?.dataset?.value, + datasourceType, JSON.stringify(loadedDatasets), ]); @@ -874,6 +952,7 @@ const FiltersConfigForm = ( filterId={filterId} filterValues={(column: Column) => !!column.is_dttm} datasetId={datasetId} + datasourceType={datasourceType} onChange={column => { // We need reset default value when column changed setNativeFilterFieldValues(form, filterId, { @@ -1062,16 +1141,23 @@ const FiltersConfigForm = ( initialValue={ datasetDetails ? { - label: DatasetSelectLabel({ - id: datasetDetails.id, - table_name: datasetDetails.table_name, - schema: datasetDetails.schema, - database: { - database_name: - datasetDetails.database.database_name, - }, - }), + label: datasetDetails.database + ? DatasetSelectLabel({ + id: datasetDetails.id, + table_name: datasetDetails.table_name, + schema: datasetDetails.schema, + database: { + database_name: + datasetDetails.database.database_name, + }, + }) + : (datasetDetails.table_name ?? + datasetDetails.id), value: datasetDetails.id, + kind: + datasourceType === DatasourceType.SemanticView + ? 'semantic_view' + : undefined, } : undefined } @@ -1090,11 +1176,20 @@ const FiltersConfigForm = ( onChange={(value: { label: string | React.ReactNode; value: number; + kind?: string; }) => { - if (value.value !== datasetId) { + const newDatasourceType = + value.kind === 'semantic_view' + ? DatasourceType.SemanticView + : DatasourceType.Table; + if ( + value.value !== datasetId || + newDatasourceType !== datasourceType + ) { setNativeFilterFieldValues(form, filterId, { dataset: value, datasetInfo: value, + datasourceType: newDatasourceType, defaultDataMask: null, column: null, }); @@ -1650,6 +1745,11 @@ const FiltersConfigForm = ( : NativeFilterType.NativeFilter } /> +