Compare commits

..
Author SHA1 Message Date
Claude Code ace0fdeb55 feat(charts): validate post-processing options against operation schemas [DRAFT]
ChartDataPostProcessingOperationSchema accepted `options` as a free-form Dict;
the per-operation option schemas (aggregate, rolling, prophet, pivot, ...)
existed only for OpenAPI docs and were never wired into validation.

Add a validates_schema hook that maps each operation to its option schema and
validates `options` against it. Validation is lenient (unknown=EXCLUDE) so it
surfaces wrong types / out-of-range values on declared fields without rejecting
payloads that carry extra keys; operations without a dedicated schema are
unaffected.

DRAFT: these option schemas were never used for validation and have latent
issues (e.g. ChartDataAggregateOptionsSchema.groupby is accidentally a tuple,
so it isn't validated). Each option schema should be audited against the real
pandas_postprocessing signatures before strict (unknown=RAISE) validation is
considered, to avoid rejecting currently-valid requests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-01 21:09:21 -07:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> a183582291 chore(deps): bump markdown-to-jsx from 9.8.0 to 9.8.1 in /superset-frontend (#40316)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-05-20 22:33:08 -07:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 3acef94ef6 chore(deps): update zod requirement from ^4.4.1 to ^4.4.3 in /superset-frontend/plugins/plugin-chart-echarts (#40313)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-05-20 22:32:46 -07:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 9638eecdb1 chore(deps-dev): bump oxlint from 1.65.0 to 1.66.0 in /superset-frontend (#40318)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-05-20 22:29:45 -07:00
Evan RusackasandClaude 7e74fc4192 fix(charts): handle PostgreSQL INTERVAL type in bar and pie charts (#34513)
Co-authored-by: Claude <noreply@anthropic.com>
2026-05-20 22:26:59 -07:00
Evan RusackasandClaude Opus 4.7 cdca6f7fdc fix(sqllab): keep saved-query list working when Jinja dataset(id) references a deleted dataset (#39703)
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-20 21:19:19 -07:00
b1ca8cac6b fix(tests): fix flaky FileHandler test by awaiting LaunchQueue consumer in afterEach (#39508)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: sadpandajoe <jcli38@gmail.com>
2026-05-20 19:31:01 -07:00
Evan RusackasandClaude Code 2cd5efa627 ci(deps): bump lower bound on pip dependabot PRs (#40308)
Co-authored-by: Claude Code <noreply@anthropic.com>
2026-05-21 08:53:57 +07:00
a273fe4d62 fix(list-view): preserve user name in filter pill after navigation (#39505)
Co-authored-by: Joe Li <joe@preset.io>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-20 16:54:49 -07:00
d203f0de33 chore(sql-lab): finish SqlLab typed-dispatch migration for SaveDatasetModal (#40040)
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Claude <claude@anthropic.com>
2026-05-20 16:04:38 -07:00
a75f9b67b2 chore(superset-ui-switchboard): forward-compat fixes for TypeScript 6.0 (Phase E) (#40028)
Co-authored-by: Claude <claude@anthropic.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-20 15:37:52 -07:00
Evan RusackasandClaude Code 3f0858e35d chore(sql-lab): migrate useDispatch to useAppDispatch (#40037)
Co-authored-by: Claude Code <noreply@anthropic.com>
2026-05-20 15:36:27 -07:00
Beto Dealmeida 68c145adc3 feat(semantic layers): add metadata on additive metrics (#40279) 2026-05-20 18:29:28 -04:00
4a9aecda4a fix(dashboard-import): remap chartsInScope on import (#26338) (#40140)
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Claude <claude@anthropic.com>
2026-05-20 13:41:14 -07:00
53 changed files with 1320 additions and 1011 deletions
+5
View File
@@ -62,6 +62,11 @@ updates:
- package-ecosystem: "pip"
directory: "/"
open-pull-requests-limit: 10
# Bump the lower bound to the new version, not just widen the upper
# bound. Without this, a `sqlglot>=28.10.0, <29` constraint upgraded
# to `<30` would keep the stale lower bound forever, dragging
# transitively-resolved versions with it. See #40186 (review thread).
versioning-strategy: increase
schedule:
interval: "weekly"
labels:
@@ -92,6 +92,26 @@ class Dimension:
grain: Grain | None = None
class AggregationType(str, enum.Enum):
"""
Aggregation function applied by a metric.
Additivity (across an arbitrary set of grouping dimensions):
* ``SUM``, ``COUNT``: fully additive — sub-group sums roll up via ``sum``.
* ``MIN``, ``MAX``: roll up via ``min`` / ``max`` of sub-group values.
* ``AVG``, ``COUNT_DISTINCT``, ``OTHER``: not safely roll-uppable from
sub-aggregates without auxiliary data.
"""
SUM = "SUM"
COUNT = "COUNT"
MIN = "MIN"
MAX = "MAX"
AVG = "AVG"
COUNT_DISTINCT = "COUNT_DISTINCT"
OTHER = "OTHER"
@dataclass(frozen=True)
class Metric:
id: str
@@ -100,6 +120,7 @@ class Metric:
definition: str
description: str | None = None
aggregation: AggregationType | None = None
@dataclass(frozen=True)
+371 -420
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -191,7 +191,7 @@
"json-stringify-pretty-compact": "^2.0.0",
"lodash": "^4.18.1",
"mapbox-gl": "^3.24.0",
"markdown-to-jsx": "^9.8.0",
"markdown-to-jsx": "^9.8.1",
"match-sorter": "^8.3.0",
"memoize-one": "^5.2.1",
"mousetrap": "^1.6.5",
@@ -350,7 +350,7 @@
"lightningcss": "^1.32.0",
"mini-css-extract-plugin": "^2.10.2",
"open-cli": "^9.0.0",
"oxlint": "^1.65.0",
"oxlint": "^1.66.0",
"po2json": "^0.4.5",
"prettier": "3.8.3",
"prettier-plugin-packagejson": "^3.0.2",
@@ -95,8 +95,11 @@ class FakeMessageChannel {
const port2 = new FakeMessagePort();
port1.otherPort = port2;
port2.otherPort = port1;
this.port1 = port1;
this.port2 = port2;
// FakeMessagePort only implements the subset of MessagePort that
// Switchboard exercises; cast at the boundary so the fake satisfies
// the consumer signature without weakening the production type.
this.port1 = port1 as unknown as MessagePort;
this.port2 = port2 as unknown as MessagePort;
}
}
@@ -88,7 +88,7 @@ function isError(message: Message): message is ErrorMessage {
* Calling methods on the switchboard causes messages to be sent through the channel.
*/
export class Switchboard {
port: MessagePort;
port!: MessagePort;
name = '';
@@ -97,9 +97,9 @@ export class Switchboard {
// used to make unique ids
incrementor = 1;
debugMode: boolean;
debugMode = false;
private isInitialised: boolean;
private isInitialised = false;
constructor(params?: Params) {
if (!params) {
@@ -29,7 +29,7 @@
"acorn": "^8.16.0",
"d3-array": "^3.2.4",
"lodash": "^4.18.1",
"zod": "^4.4.1"
"zod": "^4.4.3"
},
"peerDependencies": {
"@apache-superset/core": "*",
@@ -97,7 +97,7 @@ export function createWrapper(options?: Options) {
}
if (useDnd) {
// @ts-expect-error react-dnd types not updated for React 18
// @ts-ignore react-dnd's DndProviderProps omits `children` under React 18 types
result = <DndProvider backend={HTML5Backend}>{result}</DndProvider>;
}
+12 -6
View File
@@ -1671,7 +1671,7 @@ export interface VizOptions {
export function createDatasource(
vizOptions: VizOptions,
): SqlLabThunkAction<Promise<unknown>> {
): SqlLabThunkAction<Promise<{ id: number }>> {
return (dispatch: AppDispatch) => {
dispatch(createDatasourceStarted());
const { dbId, catalog, schema, datasourceName, sql, templateParams } =
@@ -1691,9 +1691,10 @@ export function createDatasource(
}),
})
.then(({ json }) => {
dispatch(createDatasourceSuccess(json as { id: number }));
const result = json as { id: number };
dispatch(createDatasourceSuccess(result));
return Promise.resolve(json);
return result;
})
.catch(error => {
getClientErrorObject(error).then(e => {
@@ -1712,7 +1713,7 @@ export function createDatasource(
export function createCtasDatasource(
vizOptions: Record<string, unknown>,
): SqlLabThunkAction<Promise<{ id: number }>> {
): SqlLabThunkAction<Promise<{ table_id: number }>> {
return (dispatch: AppDispatch) => {
dispatch(createDatasourceStarted());
return SupersetClient.post({
@@ -1720,9 +1721,14 @@ export function createCtasDatasource(
jsonPayload: vizOptions,
})
.then(({ json }) => {
dispatch(createDatasourceSuccess(json.result));
const result = json.result as { table_id: number };
// The endpoint's `result.table_id` IS the dataset id; normalize so
// createDatasourceSuccess's `${data.id}__table` resolves correctly.
// Without this, the CTAS Explore button silently produced
// `"undefined__table"` because `result.id` doesn't exist.
dispatch(createDatasourceSuccess({ id: result.table_id }));
return json.result;
return result;
})
.catch(() => {
const errorMsg = t('An error occurred while creating the data source');
@@ -19,7 +19,8 @@
import { useRef, useEffect, FC, useMemo } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { logging } from '@apache-superset/core/utils';
import {
SqlLabRootState,
@@ -86,7 +87,7 @@ const EditorAutoSync: FC = () => {
const editorTabLastUpdatedAt = useSelector<SqlLabRootState, number>(
state => state.sqlLab.editorTabLastUpdatedAt,
);
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const lastSavedTimestampRef = useRef<number>(editorTabLastUpdatedAt);
const currentQueryEditorId = useSelector<SqlLabRootState, string>(
@@ -17,7 +17,8 @@
* under the License.
*/
import { useState, useEffect, useRef, useCallback, useMemo } from 'react';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { usePrevious } from '@superset-ui/core';
import { css, useTheme } from '@apache-superset/core/theme';
import { Global } from '@emotion/react';
@@ -136,7 +137,7 @@ const EditorWrapper = ({
height,
hotkeys,
}: EditorWrapperProps) => {
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const queryEditor = useQueryEditor(queryEditorId, [
'id',
'dbId',
@@ -17,7 +17,8 @@
* under the License.
*/
import { useEffect, useMemo, useRef } from 'react';
import { useDispatch, useStore } from 'react-redux';
import { useStore } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { t } from '@apache-superset/core/translation';
import { getExtensionsRegistry } from '@superset-ui/core';
@@ -68,7 +69,7 @@ export function useKeywords(
catalog,
schema,
});
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const hasFetchedKeywords = useRef(false);
// skipFetch is used to prevent re-evaluating memoized keywords
// due to updated api results by skip flag
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
import { useSelector, useDispatch } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { t } from '@apache-superset/core/translation';
import { JsonObject, VizType } from '@superset-ui/core';
import { VizType } from '@superset-ui/core';
import {
createCtasDatasource,
addInfoToast,
@@ -45,7 +46,7 @@ const ExploreCtasResultsButton = ({
const errorMessage = useSelector(
(state: SqlLabRootState) => state.sqlLab.errorMessage,
);
const dispatch = useDispatch<(dispatch: any) => Promise<JsonObject>>();
const dispatch = useAppDispatch();
const buildVizOptions = {
table_name: table,
@@ -56,7 +57,7 @@ const ExploreCtasResultsButton = ({
const visualize = () => {
dispatch(createCtasDatasource(buildVizOptions))
.then((data: { table_id: number }) => {
.then(data => {
const formData = {
datasource: `${data.table_id}__table`,
metrics: ['count'],
@@ -17,7 +17,8 @@
* under the License.
*/
import { useEffect, useState } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import URI from 'urijs';
import { pick } from 'lodash';
import { useComponentDidUpdate } from '@superset-ui/core';
@@ -49,7 +50,7 @@ const PopEditorTab: React.FC<{ children?: React.ReactNode }> = ({
({ sqlLab: { tabHistory } }) => tabHistory.slice(-1)[0],
);
const [updatedUrl, setUpdatedUrl] = useState<string>(SQL_LAB_URL);
const dispatch = useDispatch();
const dispatch = useAppDispatch();
useComponentDidUpdate(() => {
setQueryEditorId(assigned => assigned ?? activeQueryEditorId);
if (activeQueryEditorId) {
@@ -17,7 +17,8 @@
* under the License.
*/
import { useRef } from 'react';
import { useSelector, useDispatch } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { isObject } from 'lodash';
import rison from 'rison';
import {
@@ -82,7 +83,7 @@ function QueryAutoRefresh({
.map(({ id }) => id),
),
);
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const checkForRefresh = () => {
const shouldRequestChecking = shouldCheckForQueries(queries);
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
import { useDispatch } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { t } from '@apache-superset/core/translation';
import { Dropdown, Button } from '@superset-ui/core/components';
import { Menu } from '@superset-ui/core/components/Menu';
@@ -75,7 +75,7 @@ const QueryLimitSelect = ({
maxRow,
defaultQueryLimit,
}: QueryLimitSelectProps) => {
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const queryEditor = useQueryEditor(queryEditorId, ['id', 'queryLimit']);
const queryLimit = queryEditor.queryLimit || defaultQueryLimit;
@@ -30,7 +30,8 @@ import ProgressBar from '@superset-ui/core/components/ProgressBar';
import { t } from '@apache-superset/core/translation';
import { QueryResponse, QueryState } from '@superset-ui/core';
import { useTheme } from '@apache-superset/core/theme';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import {
queryEditorSetSql,
@@ -92,7 +93,7 @@ const QueryTable = ({
latestQueryId,
}: QueryTableProps) => {
const theme = useTheme();
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const [selectedQuery, setSelectedQuery] = useState<QueryResponse | null>(
null,
);
@@ -27,7 +27,8 @@ import {
} from 'react';
import AutoSizer from 'react-virtualized-auto-sizer';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { useHistory } from 'react-router-dom';
import { pick } from 'lodash';
import {
@@ -231,7 +232,7 @@ const ResultSet = ({
canCopyClipboardSqlLab: canCopyClipboard,
} = usePermissions();
const history = useHistory();
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const logAction = useLogAction({ queryId, sqlEditorId: query.sqlEditorId });
const { showConfirm, ConfirmModal } = useConfirmModal();
@@ -16,8 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
import * as reactRedux from 'react-redux';
import { act } from 'react';
import { act, type ComponentProps } from 'react';
import {
cleanup,
fireEvent,
@@ -40,6 +39,19 @@ const mockedProps = {
datasource: testQuery,
};
// Render with the SqlLab user fixture preloaded into the mock store so the
// component's useSelector(state => state.user) returns a useful value.
// Previously this test used jest.spyOn(reactRedux, 'useSelector') to inject
// the user directly, which can't intercept calls routed through the typed
// useAppSelector hook.
const renderModal = (
props: Partial<ComponentProps<typeof SaveDatasetModal>> = {},
) =>
render(<SaveDatasetModal {...mockedProps} {...props} />, {
useRedux: true,
initialState: { user },
});
fetchMock.get('glob:*/api/v1/dataset/?*', {
result: mockdatasets,
dataset_count: 3,
@@ -47,17 +59,17 @@ fetchMock.get('glob:*/api/v1/dataset/?*', {
jest.useFakeTimers({ advanceTimers: true });
// Mock the user
const useSelectorMock = jest.spyOn(reactRedux, 'useSelector');
beforeEach(() => {
useSelectorMock.mockClear();
cleanup();
});
// Mock the createDatasource action
const useDispatchMock = jest.spyOn(reactRedux, 'useDispatch');
// Mock createDatasource to return a thunk that resolves with the dataset's
// new id. The test's mock store includes redux-thunk middleware (from RTK's
// getDefaultMiddleware), so dispatch(createDatasource(...)) properly unwraps
// the thunk and the production code's .then((data) => clearDatasetCache(data.id))
// chain receives `{ id: 123 }`. Individual tests can override per-call as needed.
jest.mock('src/SqlLab/actions/sqlLab', () => ({
createDatasource: jest.fn(),
createDatasource: jest.fn(() => () => Promise.resolve({ id: 123 })),
}));
jest.mock('src/explore/exploreUtils/formData', () => ({
postFormData: jest.fn(),
@@ -70,7 +82,7 @@ jest.mock('src/utils/cachedSupersetGet', () => ({
// eslint-disable-next-line no-restricted-globals -- TODO: Migrate from describe blocks
describe('SaveDatasetModal', () => {
test('renders a "Save as new" field', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
const saveRadioBtn = screen.getByRole('radio', {
name: /save as new/i,
@@ -87,7 +99,7 @@ describe('SaveDatasetModal', () => {
});
test('renders an "Overwrite existing" field', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
const overwriteRadioBtn = screen.getByRole('radio', {
name: /overwrite existing/i,
@@ -103,20 +115,20 @@ describe('SaveDatasetModal', () => {
});
test('renders a close button', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
expect(screen.getByRole('button', { name: /close/i })).toBeInTheDocument();
});
test('renders a save button when "Save as new" is selected', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
// "Save as new" is selected when the modal opens by default
expect(screen.getByRole('button', { name: /save/i })).toBeInTheDocument();
});
test('renders an overwrite button when "Overwrite existing" is selected', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
// Click the overwrite radio button to reveal the overwrite confirmation and back buttons
const overwriteRadioBtn = screen.getByRole('radio', {
@@ -130,8 +142,7 @@ describe('SaveDatasetModal', () => {
});
test('renders the overwrite button as disabled until an existing dataset is selected', async () => {
useSelectorMock.mockReturnValue({ ...user });
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
// Click the overwrite radio button
const overwriteRadioBtn = screen.getByRole('radio', {
@@ -168,8 +179,7 @@ describe('SaveDatasetModal', () => {
});
test('renders a confirm overwrite screen when overwrite is clicked', async () => {
useSelectorMock.mockReturnValue({ ...user });
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
// Click the overwrite radio button
const overwriteRadioBtn = screen.getByRole('radio', {
@@ -215,11 +225,7 @@ describe('SaveDatasetModal', () => {
});
test('sends the schema when creating the dataset', async () => {
const dummyDispatch = jest.fn().mockResolvedValue({});
useDispatchMock.mockReturnValue(dummyDispatch);
useSelectorMock.mockReturnValue({ ...user });
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
const inputFieldText = screen.getByDisplayValue(/unimportant/i);
fireEvent.change(inputFieldText, { target: { value: 'my dataset' } });
@@ -240,17 +246,9 @@ describe('SaveDatasetModal', () => {
});
test('sends the catalog when creating the dataset', async () => {
const dummyDispatch = jest.fn().mockResolvedValue({});
useDispatchMock.mockReturnValue(dummyDispatch);
useSelectorMock.mockReturnValue({ ...user });
render(
<SaveDatasetModal
{...mockedProps}
datasource={{ ...mockedProps.datasource, catalog: 'public' }}
/>,
{ useRedux: true },
);
renderModal({
datasource: { ...mockedProps.datasource, catalog: 'public' },
});
const inputFieldText = screen.getByDisplayValue(/unimportant/i);
fireEvent.change(inputFieldText, { target: { value: 'my dataset' } });
@@ -271,7 +269,7 @@ describe('SaveDatasetModal', () => {
});
test('does not renders a checkbox button when template processing is disabled', () => {
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
expect(screen.queryByRole('checkbox')).not.toBeInTheDocument();
});
@@ -280,7 +278,7 @@ describe('SaveDatasetModal', () => {
global.featureFlags = {
[FeatureFlag.EnableTemplateProcessing]: true,
};
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
expect(screen.getByRole('checkbox')).toBeInTheDocument();
});
@@ -289,15 +287,11 @@ describe('SaveDatasetModal', () => {
global.featureFlags = {
[FeatureFlag.EnableTemplateProcessing]: true,
};
const propsWithTemplateParam = {
...mockedProps,
renderModal({
datasource: {
...testQuery,
templateParams: JSON.stringify({ my_param: 12 }),
},
};
render(<SaveDatasetModal {...propsWithTemplateParam} />, {
useRedux: true,
});
const inputFieldText = screen.getByDisplayValue(/unimportant/i);
fireEvent.change(inputFieldText, { target: { value: 'my dataset' } });
@@ -324,15 +318,11 @@ describe('SaveDatasetModal', () => {
global.featureFlags = {
[FeatureFlag.EnableTemplateProcessing]: true,
};
const propsWithTemplateParam = {
...mockedProps,
renderModal({
datasource: {
...testQuery,
templateParams: JSON.stringify({ my_param: 12 }),
},
};
render(<SaveDatasetModal {...propsWithTemplateParam} />, {
useRedux: true,
});
const inputFieldText = screen.getByDisplayValue(/unimportant/i);
fireEvent.change(inputFieldText, { target: { value: 'my dataset' } });
@@ -393,19 +383,11 @@ describe('SaveDatasetModal', () => {
.spyOn(SupersetClient, 'put')
.mockResolvedValue({ json: { result: { id: 0 } } } as any);
const dummyDispatch = jest.fn().mockResolvedValue({});
useDispatchMock.mockReturnValue(dummyDispatch);
useSelectorMock.mockReturnValue({ ...user });
const propsWithTemplateParam = {
...mockedProps,
renderModal({
datasource: {
...testQuery,
templateParams: JSON.stringify({ my_param: 12, _filters: 'foo' }),
},
};
render(<SaveDatasetModal {...propsWithTemplateParam} />, {
useRedux: true,
});
// Check the "Include Template Parameters" checkbox
@@ -443,19 +425,11 @@ describe('SaveDatasetModal', () => {
.spyOn(SupersetClient, 'put')
.mockResolvedValue({ json: { result: { id: 0 } } } as any);
const dummyDispatch = jest.fn().mockResolvedValue({});
useDispatchMock.mockReturnValue(dummyDispatch);
useSelectorMock.mockReturnValue({ ...user });
const propsWithTemplateParam = {
...mockedProps,
renderModal({
datasource: {
...testQuery,
templateParams: JSON.stringify({ my_param: 12 }),
},
};
render(<SaveDatasetModal {...propsWithTemplateParam} />, {
useRedux: true,
});
// Do NOT check the "Include Template Parameters" checkbox
@@ -489,12 +463,9 @@ describe('SaveDatasetModal', () => {
'postFormData',
);
const dummyDispatch = jest.fn().mockResolvedValue({ id: 123 });
useDispatchMock.mockReturnValue(dummyDispatch);
useSelectorMock.mockReturnValue({ ...user });
postFormData.mockResolvedValue('chart_key_123');
render(<SaveDatasetModal {...mockedProps} />, { useRedux: true });
renderModal();
const inputFieldText = screen.getByDisplayValue(/unimportant/i);
fireEvent.change(inputFieldText, { target: { value: 'my dataset' } });
@@ -34,7 +34,6 @@ import { t } from '@apache-superset/core/translation';
import {
SupersetClient,
JsonResponse,
JsonObject,
QueryResponse,
QueryFormData,
VizType,
@@ -44,16 +43,14 @@ import {
} from '@superset-ui/core';
import { styled } from '@apache-superset/core/theme';
import { extendedDayjs as dayjs } from '@superset-ui/core/utils/dates';
import { useSelector, useDispatch } from 'react-redux';
import { useAppDispatch, useAppSelector } from 'src/views/store';
import rison from 'rison';
import { createDatasource } from 'src/SqlLab/actions/sqlLab';
import { addDangerToast } from 'src/components/MessageToasts/actions';
import { UserWithPermissionsAndRoles as User } from 'src/types/bootstrapTypes';
import {
DatasetRadioState,
EXPLORE_CHART_DEFAULT,
DatasetOwner,
SqlLabRootState,
} from 'src/SqlLab/types';
import { mountExploreUrl } from 'src/explore/exploreUtils';
import { postFormData } from 'src/explore/exploreUtils/formData';
@@ -221,7 +218,7 @@ export const SaveDatasetModal = ({
openWindow = true,
formData = {},
}: SaveDatasetModalProps) => {
const defaultVizType = useSelector<SqlLabRootState, string>(
const defaultVizType = useAppSelector(
state => state.common?.conf?.DEFAULT_VIZ_TYPE || VizType.Table,
);
@@ -240,8 +237,8 @@ export const SaveDatasetModal = ({
>(undefined);
const [loading, setLoading] = useState<boolean>(false);
const user = useSelector<SqlLabRootState, User>(state => state.user);
const dispatch = useDispatch<(dispatch: any) => Promise<JsonObject>>();
const user = useAppSelector(state => state.user);
const dispatch = useAppDispatch();
const [includeTemplateParameters, setIncludeTemplateParameters] =
useState(false);
@@ -17,7 +17,8 @@
* under the License.
*/
import { createRef, useCallback, useMemo } from 'react';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { nanoid } from 'nanoid';
import Tabs from '@superset-ui/core/components/Tabs';
import { t } from '@apache-superset/core/translation';
@@ -105,7 +106,7 @@ const SouthPane = ({
const { id, tabViewId } = useQueryEditor(queryEditorId, ['tabViewId']);
const editorId = tabViewId ?? id;
const theme = useTheme();
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const viewItems = views.getViews(ViewLocations.sqllab.panels) || [];
const { offline, tables } = useSelector(
({ sqlLab: { offline, tables } }: SqlLabRootState) => ({
@@ -30,7 +30,8 @@ import {
import type { editors } from '@apache-superset/core';
import useEffectEvent from 'src/hooks/useEffectEvent';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import AutoSizer from 'react-virtualized-auto-sizer';
import { t } from '@apache-superset/core/translation';
import {
@@ -237,7 +238,7 @@ const SqlEditor: FC<Props> = ({
scheduleQueryWarning,
}) => {
const theme = useTheme();
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const { database, latestQuery, currentQueryEditorId, hasSqlStatement } =
useSelector<
@@ -17,7 +17,7 @@
* under the License.
*/
import { useCallback, useState } from 'react';
import { useDispatch } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { resetState } from 'src/SqlLab/actions/sqlLab';
import {
@@ -69,7 +69,7 @@ const SqlEditorLeftBar = ({ queryEditorId }: SqlEditorLeftBarProps) => {
const { db, catalog, schema, onDbChange, onCatalogChange, onSchemaChange } =
dbSelectorProps;
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const shouldShowReset = window.location.search === '?reset=1';
// Modal state for Database/Catalog/Schema selector
@@ -19,7 +19,8 @@
import { useMemo, FC } from 'react';
import { bindActionCreators } from 'redux';
import { useSelector, useDispatch, shallowEqual } from 'react-redux';
import { useSelector, shallowEqual } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { MenuDotsDropdown } from '@superset-ui/core/components';
import { Menu, MenuItemType } from '@superset-ui/core/components/Menu';
import { t } from '@apache-superset/core/translation';
@@ -90,7 +91,7 @@ const SqlEditorTabHeader: FC<Props> = ({ queryEditor }) => {
);
const StatusIcon = queryState ? STATE_ICONS[queryState] : STATE_ICONS.running;
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const actions = useMemo(
() =>
bindActionCreators(
@@ -17,7 +17,8 @@
* under the License.
*/
import { useEffect, useCallback, useMemo, useState } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { SqlLabRootState } from 'src/SqlLab/types';
import {
@@ -41,7 +42,7 @@ export default function useDatabaseSelector(queryEditorId: string) {
SqlLabRootState,
SqlLabRootState['sqlLab']['databases']
>(({ sqlLab }) => sqlLab.databases);
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const queryEditor = useQueryEditor(queryEditorId, [
'dbId',
'catalog',
@@ -17,7 +17,8 @@
* under the License.
*/
import { useState, useRef, useEffect } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import type { QueryEditor, SqlLabRootState, Table } from 'src/SqlLab/types';
import {
ButtonGroup,
@@ -75,7 +76,7 @@ const Fade = styled.div`
const TableElement = ({ table, ...props }: TableElementProps) => {
const { dbId, catalog, schema, name, expanded, id } = table;
const theme = useTheme();
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const {
currentData: tableMetadata,
isSuccess: isMetadataSuccess,
@@ -25,7 +25,8 @@ import {
type ChangeEvent,
useMemo,
} from 'react';
import { useSelector, useDispatch, shallowEqual } from 'react-redux';
import { useSelector, shallowEqual } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { styled, css, useTheme } from '@apache-superset/core/theme';
import { t } from '@apache-superset/core/translation';
import AutoSizer from 'react-virtualized-auto-sizer';
@@ -163,7 +164,7 @@ const savePinnedSchemasToStorage = (
};
const TableExploreTree: React.FC<Props> = ({ queryEditorId }) => {
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const theme = useTheme();
const treeRef = useRef<TreeApi<TreeNodeData>>(null);
const tables = useSelector(
@@ -17,7 +17,7 @@
* under the License.
*/
import { useMemo, useReducer, useCallback } from 'react';
import { useDispatch } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { t } from '@apache-superset/core/translation';
import {
Table,
@@ -130,7 +130,7 @@ const useTreeData = ({
catalog,
pinnedTables,
}: UseTreeDataParams): UseTreeDataResult => {
const reduxDispatch = useDispatch();
const reduxDispatch = useAppDispatch();
// Schema data from API
const {
currentData: schemaData,
@@ -17,7 +17,8 @@
* under the License.
*/
import { type FC, useCallback, useMemo, useRef, useState } from 'react';
import { shallowEqual, useDispatch, useSelector } from 'react-redux';
import { shallowEqual, useSelector } from 'react-redux';
import { useAppDispatch } from 'src/views/store';
import { nanoid } from 'nanoid';
import { t } from '@apache-superset/core/translation';
import { ClientErrorObject, getExtensionsRegistry } from '@superset-ui/core';
@@ -110,7 +111,7 @@ const renderWell = (partitions: TableMetaData['partitions']) => {
};
const TablePreview: FC<Props> = ({ dbId, catalog, schema, tableName }) => {
const dispatch = useDispatch();
const dispatch = useAppDispatch();
const theme = useTheme();
const [databaseName, backend, disableDataPreview] = useSelector<
SqlLabRootState,
@@ -0,0 +1,267 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createRef } from 'react';
import {
render,
screen,
selectOption,
waitFor,
} from 'spec/helpers/testing-library';
import { ListViewFilterOperator } from '../types';
import UIFilters from './index';
import SelectFilter from './Select';
import type { FilterHandler } from './types';
const mockUpdateFilterValue = jest.fn();
beforeEach(() => {
mockUpdateFilterValue.mockClear();
});
test('select filter with ReactNode label uses option title when serializing selection', async () => {
// Regression for sc-104554: the chart-list Owner filter renders options
// with ReactNode labels (name + email). The value passed to
// updateFilterValue is serialized into URL / filter state and re-used to
// render the filter pill on return. It must carry the plain-text name
// (from `title`) and not fall back to the numeric user id.
const ReactNodeLabel = (
<div>
<span>John Doe</span>
<span>john@example.com</span>
</div>
);
const fetchSelects = jest.fn().mockResolvedValue({
data: [
{
label: ReactNodeLabel,
value: 42,
title: 'John Doe',
},
],
totalCount: 1,
});
const filters = [
{
Header: 'Owner',
key: 'owner',
id: 'owners',
input: 'select' as const,
operator: ListViewFilterOperator.RelationManyMany,
unfilteredLabel: 'All',
fetchSelects,
paginate: true,
},
];
render(
<UIFilters
filters={filters}
internalFilters={[]}
updateFilterValue={mockUpdateFilterValue}
/>,
);
await selectOption('John Doe', 'Owner');
await waitFor(() => {
expect(mockUpdateFilterValue).toHaveBeenCalledWith(0, {
label: 'John Doe',
value: 42,
});
});
});
test('select filter falls back to stringified value when no string label or title is available', async () => {
const fetchSelects = jest.fn().mockResolvedValue({
data: [
{
label: <span>123</span>,
value: 123,
},
],
totalCount: 1,
});
const filters = [
{
Header: 'Something',
key: 'something',
id: 'something',
input: 'select' as const,
operator: ListViewFilterOperator.RelationOneMany,
unfilteredLabel: 'All',
fetchSelects,
},
];
render(
<UIFilters
filters={filters}
internalFilters={[]}
updateFilterValue={mockUpdateFilterValue}
/>,
);
await selectOption('123', 'Something');
await waitFor(() => {
expect(mockUpdateFilterValue).toHaveBeenCalledWith(0, {
label: '123',
value: 123,
});
});
});
test('plain select with string label passes label through unchanged', async () => {
// Happy-path coverage for the typeof-string branch in onChange, exercised
// through the non-async Select wrapper (selects array, no fetchSelects).
const filters = [
{
Header: 'Status',
key: 'status',
id: 'status',
input: 'select' as const,
operator: ListViewFilterOperator.Equals,
unfilteredLabel: 'All',
selects: [
{ label: 'Published', value: 7 },
{ label: 'Draft', value: 8 },
],
},
];
render(
<UIFilters
filters={filters}
internalFilters={[]}
updateFilterValue={mockUpdateFilterValue}
/>,
);
await selectOption('Published', 'Status');
await waitFor(() => {
expect(mockUpdateFilterValue).toHaveBeenCalledWith(0, {
label: 'Published',
value: 7,
});
});
});
test('plain select with ReactNode label uses option title when serializing selection', async () => {
// Parallel coverage to the AsyncSelect ReactNode-with-title test, against
// the non-async Select wrapper. Guards against the two wrappers ever
// diverging on antd's two-arg onChange shape.
const ReactNodeLabel = (
<div>
<span>Jane Roe</span>
<span>jane@example.com</span>
</div>
);
const filters = [
{
Header: 'Owner',
key: 'owner',
id: 'owners',
input: 'select' as const,
operator: ListViewFilterOperator.RelationManyMany,
unfilteredLabel: 'All',
selects: [{ label: ReactNodeLabel, value: 99, title: 'Jane Roe' }],
},
];
render(
<UIFilters
filters={filters}
internalFilters={[]}
updateFilterValue={mockUpdateFilterValue}
/>,
);
await selectOption('Jane Roe', 'Owner');
await waitFor(() => {
expect(mockUpdateFilterValue).toHaveBeenCalledWith(0, {
label: 'Jane Roe',
value: 99,
});
});
});
test('clearFilter notifies onSelect with undefined and isClear=true', () => {
// The isClear flag is what allows the parent (Filters/index) to suppress
// onFilterUpdate side-effects when the user clears the filter rather than
// picking a new value. Lock that contract in.
const mockOnSelect = jest.fn();
const ref = createRef<FilterHandler>();
render(
<SelectFilter
Header="Owner"
initialValue={{ label: 'John Doe', value: 42 }}
onSelect={mockOnSelect}
selects={[{ label: 'John Doe', value: 42, title: 'John Doe' }]}
ref={ref}
/>,
);
ref.current?.clearFilter();
expect(mockOnSelect).toHaveBeenCalledWith(undefined, true);
});
test('rehydrates filter pill from initialValue with plain-string label', async () => {
// The user-visible regression: after URL/state rehydration the filter pill
// must render the human-readable name, not the numeric user id. The fix
// ensures the persisted label is a string; this test asserts that string
// is what surfaces in the rendered combobox selection.
const filters = [
{
Header: 'Owner',
key: 'owner',
id: 'owners',
input: 'select' as const,
operator: ListViewFilterOperator.RelationManyMany,
unfilteredLabel: 'All',
fetchSelects: jest.fn().mockResolvedValue({ data: [], totalCount: 0 }),
paginate: true,
},
];
render(
<UIFilters
filters={filters}
internalFilters={[
{
id: 'owners',
operator: ListViewFilterOperator.RelationManyMany,
value: { label: 'John Doe', value: 42 },
},
]}
updateFilterValue={mockUpdateFilterValue}
/>,
);
await waitFor(() => {
expect(screen.getByText('John Doe')).toBeInTheDocument();
});
});
@@ -58,14 +58,22 @@ function SelectFilter(
) {
const [selectedOption, setSelectedOption] = useState(initialValue);
const onChange = (selected: SelectOption) => {
const onChange = (selected: SelectOption, option?: SelectOption) => {
// antd's `onChange` (with `labelInValue`) passes the `{label, value}`
// labeled-value as the first arg and the full option (which carries
// `title` and any other fields) as the second. Options may supply a
// ReactNode label (e.g. OwnerSelectLabel for the chart list Owner
// filter). Since this object is serialized into the URL and rehydrated
// as the filter pill on return, we need a plain string. Prefer `title`
// (set by callers to the human-readable name) before falling back to
// the value.
onSelect(
selected
? {
label:
typeof selected.label === 'string'
? selected.label
: String(selected.value),
: (option?.title ?? String(selected.value)),
value: selected.value,
}
: undefined,
@@ -26,6 +26,10 @@ export interface SortColumn {
export interface SelectOption {
label: ReactNode;
value: any;
// Plain-text representation of the option. Callers should set this when
// `label` is a ReactNode so that the option can be serialized (e.g. into
// URL filter state) without losing the human-readable name.
title?: string;
[key: string]: unknown;
}
@@ -622,15 +622,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
const [editNewDb, setEditNewDb] = useState<boolean>(false);
const [isLoading, setLoading] = useState<boolean>(false);
const [testInProgress, setTestInProgress] = useState<boolean>(false);
// Stable id sent on every "Test Connection" call so that the backend can
// correlate the wizard's OAuth2 dance across requests. Generated once per
// modal session — keeping the same value lets the second test_connection
// call (after the user finishes the OAuth2 dance) find the cached token.
const oauth2TabIdRef = useRef<string>(
typeof crypto !== 'undefined' && crypto.randomUUID
? crypto.randomUUID()
: Math.random().toString(36).slice(2),
);
const [passwords, setPasswords] = useState<Record<string, string>>({});
const [sshTunnelPasswords, setSSHTunnelPasswords] = useState<
Record<string, string>
@@ -736,11 +727,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
}, [setValidationErrors, setHasValidated, clearError]);
// Test Connection logic
// keep the latest ``testConnection`` callable accessible to long-lived
// listeners (BroadcastChannel/storage) without making them depend on every
// render of the modal.
const testConnectionRef = useRef<() => void>(() => {});
const testConnection = () => {
handleClearValidationErrors();
if (!db?.sqlalchemy_uri) {
@@ -762,7 +748,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
server_port: Number(db.ssh_tunnel!.server_port),
}
: undefined,
oauth2_tab_id: oauth2TabIdRef.current,
};
setTestInProgress(true);
testDatabaseConnection(
@@ -779,47 +764,6 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
},
);
};
testConnectionRef.current = testConnection;
// Re-run "Test Connection" automatically when the OAuth2 dance completes
// in the popup tab. The dance posts a message on the ``oauth`` broadcast
// channel (and a localStorage event for cross-context delivery) carrying
// the wizard's own tab_id; we react only to our own.
useEffect(() => {
const tabId = oauth2TabIdRef.current;
const handleComplete = (incomingTabId?: string) => {
if (incomingTabId === tabId) {
testConnectionRef.current();
}
};
const channel =
typeof BroadcastChannel !== 'undefined'
? new BroadcastChannel('oauth')
: null;
if (channel) {
channel.onmessage = event => handleComplete(event.data?.tabId);
}
const handleStorage = (event: StorageEvent) => {
if (event.key !== 'oauth2_auth_complete' || !event.newValue) {
return;
}
try {
const payload = JSON.parse(event.newValue) as { tabId?: string };
handleComplete(payload.tabId);
} catch {
/* ignore */
}
};
window.addEventListener('storage', handleStorage);
return () => {
window.removeEventListener('storage', handleStorage);
channel?.close();
};
}, []);
const getPlaceholder = (field: string) => {
if (field === 'database') {
@@ -117,6 +117,7 @@ type LaunchQueue = {
const pendingTimerIds = new Set<ReturnType<typeof setTimeout>>();
const MAX_CONSUMER_POLL_ATTEMPTS = 50;
const consumerPromises: Promise<void>[] = [];
// Defer the consumer call to a macrotask so it doesn't fire synchronously inside
// the component's useEffect — calling it inline deadlocks Jest because the
@@ -131,7 +132,11 @@ const setupLaunchQueue = (fileHandle: MockFileHandle | null = null) => {
if (fileHandle) {
const id = setTimeout(() => {
pendingTimerIds.delete(id);
consumer({ files: [fileHandle] });
consumerPromises.push(
Promise.resolve(consumer({ files: [fileHandle] })).then(
() => undefined,
),
);
}, 0);
pendingTimerIds.add(id);
}
@@ -165,9 +170,19 @@ beforeEach(() => {
.launchQueue;
});
afterEach(() => {
afterEach(async () => {
pendingTimerIds.forEach(id => clearTimeout(id));
pendingTimerIds.clear();
if (consumerPromises.length > 0) {
const results = await Promise.allSettled(consumerPromises);
results.forEach(r => {
if (r.status === 'rejected') {
// eslint-disable-next-line no-console
console.warn('LaunchQueue consumer rejected:', r.reason);
}
});
consumerPromises.length = 0;
}
delete (window as unknown as Window & { launchQueue?: LaunchQueue })
.launchQueue;
});
+11 -2
View File
@@ -22,12 +22,13 @@ import {
createListenerMiddleware,
StoreEnhancer,
} from '@reduxjs/toolkit';
import type { AnyAction } from 'redux';
import {
useDispatch,
useSelector,
type TypedUseSelectorHook,
} from 'react-redux';
import thunk from 'redux-thunk';
import thunk, { type ThunkDispatch } from 'redux-thunk';
import { api } from 'src/hooks/apiResources/queryApi';
import messageToastReducer from 'src/components/MessageToasts/reducers';
import charts from 'src/components/Chart/chartReducer';
@@ -188,6 +189,14 @@ export type RootState = ReturnType<typeof store.getState>;
// thunks resolve correctly), and `useAppSelector` infers `RootState` without
// callers having to annotate every selector. Required ahead of the
// react-redux v8+ bump, which tightens dispatch typing — see #39927.
export type AppDispatch = typeof store.dispatch;
//
// AppDispatch is declared as ThunkDispatch & store.dispatch rather than
// `typeof store.dispatch` because Superset annotates getMiddleware as
// ConfigureStoreOptions['middleware'], which erases the middleware tuple type
// and leaves store.dispatch typed as Dispatch<AnyAction>. The intersection
// restores thunk support without requiring a wider refactor of the middleware
// setup.
export type AppDispatch = ThunkDispatch<RootState, undefined, AnyAction> &
typeof store.dispatch;
export const useAppDispatch: () => AppDispatch = useDispatch;
export const useAppSelector: TypedUseSelectorHook<RootState> = useSelector;
+45 -1
View File
@@ -22,7 +22,15 @@ from typing import Any, TYPE_CHECKING
from flask import current_app
from flask_babel import gettext as _
from marshmallow import EXCLUDE, fields, post_load, Schema, validate
from marshmallow import (
EXCLUDE,
fields,
post_load,
Schema,
validate,
validates_schema,
ValidationError,
)
from marshmallow.validate import Length, Range
from marshmallow_union import Union
@@ -937,6 +945,42 @@ class ChartDataPostProcessingOperationSchema(Schema):
},
)
# Map post-processing operation -> its options schema, for operations that
# declare one. Operations without a dedicated schema are not structurally
# validated here.
_OPTIONS_SCHEMAS: dict[str, type[Schema]] = {
"aggregate": ChartDataAggregateOptionsSchema,
"rolling": ChartDataRollingOptionsSchema,
"select": ChartDataSelectOptionsSchema,
"sort": ChartDataSortOptionsSchema,
"contribution": ChartDataContributionOptionsSchema,
"prophet": ChartDataProphetOptionsSchema,
"boxplot": ChartDataBoxplotOptionsSchema,
"pivot": ChartDataPivotOptionsSchema,
"geohash_decode": ChartDataGeohashDecodeOptionsSchema,
"geohash_encode": ChartDataGeohashEncodeOptionsSchema,
"geodetic_parse": ChartDataGeodeticParseOptionsSchema,
}
@validates_schema
def validate_options(self, data: dict[str, Any], **kwargs: Any) -> None:
"""Validate ``options`` against the operation's option schema.
Validation is lenient (unknown keys are ignored) so it surfaces wrong
types / out-of-range values on declared fields without rejecting
payloads that carry extra keys.
"""
operation = data.get("operation")
options = data.get("options")
if not isinstance(operation, str) or not isinstance(options, dict):
return
schema_cls = self._OPTIONS_SCHEMAS.get(operation)
if schema_cls is None:
return
errors = schema_cls(unknown=EXCLUDE).validate(options)
if errors:
raise ValidationError({"options": errors})
class ChartDataFilterSchema(Schema):
col = fields.Raw(
@@ -62,6 +62,21 @@ def build_uuid_to_id_map(position: dict[str, Any]) -> dict[str, int]:
}
def _remap_charts_in_scope(container: dict[str, Any], id_map: dict[int, int]) -> None:
"""Remap source-env chart IDs in ``container["chartsInScope"]`` in place.
``chartsInScope`` is a denormalized cache of the charts a filter (native
or cross-filter) currently applies to. Both surfaces share this contract,
so they share this remap. Unresolvable IDs are dropped rather than
passed through, matching the convention used for ``scope.excluded``.
"""
charts_in_scope = container.get("chartsInScope")
if isinstance(charts_in_scope, list):
container["chartsInScope"] = [
id_map[old_id] for old_id in charts_in_scope if old_id in id_map
]
def update_id_refs( # pylint: disable=too-many-locals # noqa: C901
config: dict[str, Any],
chart_ids: dict[str, int],
@@ -145,6 +160,8 @@ def update_id_refs( # pylint: disable=too-many-locals # noqa: C901
id_map[old_id] for old_id in scope_excluded if old_id in id_map
]
_remap_charts_in_scope(native_filter, id_map)
# fix display control dataset references
for customization in (
fixed.get("metadata", {}).get("chart_customization_config") or []
@@ -170,7 +187,7 @@ def update_id_refs( # pylint: disable=too-many-locals # noqa: C901
return fixed
def update_cross_filter_scoping(
def update_cross_filter_scoping( # noqa: C901
config: dict[str, Any], id_map: dict[int, int]
) -> dict[str, Any]:
# fix cross filter references
@@ -185,6 +202,9 @@ def update_cross_filter_scoping(
id_map[old_id] for old_id in scope_excluded if old_id in id_map
]
# Global cross-filter chartsInScope mirrors the native-filter case.
_remap_charts_in_scope(cross_filter_global_config, id_map)
if "chart_configuration" in (metadata := fixed.get("metadata", {})):
# Build remapped configuration in a single pass for clarity/readability.
new_chart_configuration: dict[str, Any] = {}
@@ -212,6 +232,11 @@ def update_cross_filter_scoping(
if old_id in id_map
]
# Cross-filter chartsInScope mirrors the native-filter case.
cross_filters = chart_config.get("crossFilters")
if isinstance(cross_filters, dict):
_remap_charts_in_scope(cross_filters, id_map)
new_chart_configuration[str(new_id)] = chart_config
metadata["chart_configuration"] = new_chart_configuration
+41 -119
View File
@@ -25,157 +25,66 @@ from superset.commands.database.exceptions import DatabaseNotFoundError
from superset.daos.database import DatabaseUserOAuth2TokensDAO
from superset.daos.key_value import KeyValueDAO
from superset.databases.schemas import OAuth2ProviderResponseSchema
from superset.db_engine_specs import get_engine_spec
from superset.db_engine_specs.base import BaseEngineSpec
from superset.exceptions import OAuth2Error
from superset.key_value.types import JsonKeyValueCodec, KeyValueResource
from superset.models.core import Database, DatabaseUserOAuth2Tokens
from superset.superset_typing import (
OAuth2ClientConfig,
OAuth2State,
OAuth2TokenResponse,
)
from superset.superset_typing import OAuth2State
from superset.utils.decorators import on_error, transaction
from superset.utils.oauth2 import decode_oauth2_state
# how long the pre-create token cache lives in the KV store
PRE_CREATE_TOKEN_TTL = timedelta(minutes=5)
class OAuth2StoreTokenCommand(BaseCommand):
"""
Command to store OAuth2 tokens.
Normal flow: the OAuth2 callback resolves the database via ``state.database_id``
and persists access/refresh tokens to ``database_user_oauth2_tokens``.
Pre-create flow: when ``state.database_id`` is ``None`` (the database hasn't
been saved yet typically the "Create database" wizard), the command reads
the OAuth2 client config and engine name from the KV store entry that
:meth:`BaseEngineSpec.start_oauth2_dance` stashed there, exchanges the code,
and caches the resulting access token in the same KV entry for
:func:`get_oauth2_access_token` to pick up on the retry.
Command to store OAuth2 tokens in the database.
"""
def __init__(self, parameters: OAuth2ProviderResponseSchema):
self._parameters = parameters
self._state: OAuth2State | None = None
self._database: Database | None = None
self._oauth2_config: OAuth2ClientConfig | None = None
self._engine_spec: type[BaseEngineSpec] | None = None
self._tab_uuid: UUID | None = None
@transaction(on_error=partial(on_error, reraise=OAuth2Error))
def run(self) -> DatabaseUserOAuth2Tokens | None:
def run(self) -> DatabaseUserOAuth2Tokens:
self.validate()
self._database = cast(Database, self._database)
self._state = cast(OAuth2State, self._state)
self._oauth2_config = cast(OAuth2ClientConfig, self._oauth2_config)
self._engine_spec = cast(type[BaseEngineSpec], self._engine_spec)
oauth2_config = self._database.get_oauth2_config()
if oauth2_config is None:
raise OAuth2Error("No configuration found for OAuth2")
# Look up PKCE code_verifier from KV store (RFC 7636)
code_verifier = self._pop_code_verifier()
code_verifier = None
tab_id = self._state["tab_id"]
try:
tab_uuid = UUID(tab_id)
except ValueError:
tab_uuid = None
token_response = self._engine_spec.get_oauth2_token(
self._oauth2_config,
if tab_uuid:
kv_value = KeyValueDAO.get_value(
resource=KeyValueResource.PKCE_CODE_VERIFIER,
key=tab_uuid,
codec=JsonKeyValueCodec(),
)
if kv_value:
code_verifier = kv_value.get("code_verifier")
KeyValueDAO.delete_entry(KeyValueResource.PKCE_CODE_VERIFIER, tab_uuid)
token_response = self._database.db_engine_spec.get_oauth2_token(
oauth2_config,
self._parameters["code"],
code_verifier=code_verifier,
)
if self._database is None:
# Pre-create flow: cache the access token in the KV entry the
# initial dance created. The retry of "Test Connection" will read
# it via ``get_oauth2_access_token``.
self._cache_pre_create_token(token_response)
return None
return self._persist_token(token_response)
def validate(self) -> None:
if error := self._parameters.get("error"):
raise OAuth2Error(error)
self._state = decode_oauth2_state(self._parameters["state"])
try:
self._tab_uuid = UUID(self._state["tab_id"])
except (KeyError, ValueError):
# Legacy paths may use non-UUID tab ids; we still want to support
# them when ``database_id`` is set. The pre-create path below
# requires a valid UUID.
self._tab_uuid = None
if database_id := self._state.get("database_id"):
self._database = DatabaseUserOAuth2TokensDAO.get_database(database_id)
if self._database is None:
raise DatabaseNotFoundError("Database not found")
self._oauth2_config = self._database.get_oauth2_config()
self._engine_spec = self._database.db_engine_spec
else:
if self._tab_uuid is None:
raise OAuth2Error(
"Pre-create OAuth2 callback requires a UUID tab_id",
)
cached = KeyValueDAO.get_value(
resource=KeyValueResource.OAUTH2_PRE_CREATE_TOKEN,
key=self._tab_uuid,
codec=JsonKeyValueCodec(),
)
if not cached or not cached.get("config"):
raise OAuth2Error("Pre-create OAuth2 context not found or expired")
self._oauth2_config = cast(OAuth2ClientConfig, cached["config"])
engine = self._state.get("engine") or cached.get("engine")
if not engine:
raise OAuth2Error("Pre-create OAuth2 context missing engine name")
self._engine_spec = get_engine_spec(engine)
if self._oauth2_config is None:
raise OAuth2Error("No configuration found for OAuth2")
def _pop_code_verifier(self) -> str | None:
if self._tab_uuid is None:
return None
kv_value = KeyValueDAO.get_value(
resource=KeyValueResource.PKCE_CODE_VERIFIER,
key=self._tab_uuid,
codec=JsonKeyValueCodec(),
)
if not kv_value:
return None
KeyValueDAO.delete_entry(KeyValueResource.PKCE_CODE_VERIFIER, self._tab_uuid)
return kv_value.get("code_verifier")
def _cache_pre_create_token(self, token_response: OAuth2TokenResponse) -> None:
self._state = cast(OAuth2State, self._state)
self._tab_uuid = cast(UUID, self._tab_uuid)
self._oauth2_config = cast(OAuth2ClientConfig, self._oauth2_config)
self._engine_spec = cast(type[BaseEngineSpec], self._engine_spec)
expires_on = datetime.now() + PRE_CREATE_TOKEN_TTL
KeyValueDAO.upsert_entry(
resource=KeyValueResource.OAUTH2_PRE_CREATE_TOKEN,
key=self._tab_uuid,
value={
"engine": self._engine_spec.engine,
"config": self._oauth2_config,
"user_id": self._state["user_id"],
"access_token": token_response["access_token"],
},
codec=JsonKeyValueCodec(),
expires_on=expires_on,
)
def _persist_token(
self,
token_response: OAuth2TokenResponse,
) -> DatabaseUserOAuth2Tokens:
self._state = cast(OAuth2State, self._state)
# delete old tokens
if existing := DatabaseUserOAuth2TokensDAO.find_one_or_none(
user_id=self._state["user_id"],
database_id=self._state["database_id"],
):
DatabaseUserOAuth2TokensDAO.delete([existing])
# store tokens
expiration = datetime.now() + timedelta(seconds=token_response["expires_in"])
return DatabaseUserOAuth2TokensDAO.create(
attributes={
@@ -186,3 +95,16 @@ class OAuth2StoreTokenCommand(BaseCommand):
"refresh_token": token_response.get("refresh_token"),
},
)
def validate(self) -> None:
if error := self._parameters.get("error"):
raise OAuth2Error(error)
self._state = decode_oauth2_state(self._parameters["state"])
if database := DatabaseUserOAuth2TokensDAO.get_database(
self._state["database_id"]
):
self._database = database
else:
raise DatabaseNotFoundError("Database not found")
@@ -17,7 +17,6 @@
import logging
from typing import Any, Optional
from flask import g
from flask_babel import gettext as _
from sqlalchemy.exc import DBAPIError, NoSuchModuleError
@@ -95,13 +94,6 @@ class TestConnectionDatabaseCommand(BaseCommand):
self.validate()
ex_str = ""
# Surface the wizard's tab_id (sent by the frontend) so that
# ``get_oauth2_access_token`` can find the pre-create OAuth2 token
# cached in the KV store, and so that ``start_oauth2_dance`` reuses
# this id instead of generating a new one.
if oauth2_tab_id := self._properties.get("oauth2_tab_id"):
g.oauth2_tab_id = oauth2_tab_id
url = make_url_safe(self._uri)
engine_name = url.get_backend_name()
-10
View File
@@ -623,16 +623,6 @@ class DatabaseTestConnectionSchema(DatabaseParametersSchemaMixin, Schema):
)
ssh_tunnel = fields.Nested(DatabaseSSHTunnel, allow_none=True)
oauth2_tab_id = fields.String(
metadata={
"description": (
"UUID identifying the wizard tab for the pre-create OAuth2 flow."
" Optional; when supplied, the engine will look up the pre-create"
" OAuth2 token cached under this key."
),
},
allow_none=True,
)
class TableMetadataOptionsResponse(TypedDict):
+1 -26
View File
@@ -664,10 +664,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# Prevent circular import.
from superset.daos.key_value import KeyValueDAO
# Reuse the wizard-supplied tab_id when present so the retry of
# "Test Connection" can find the same KV-cached token. Otherwise we
# generate a fresh one as before.
tab_id = getattr(g, "oauth2_tab_id", None) or str(uuid4())
tab_id = str(uuid4())
default_redirect_uri = get_oauth2_redirect_uri()
# Generate PKCE code verifier (RFC 7636)
@@ -693,9 +690,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# belongs to.
state: OAuth2State = {
# Database ID and user ID are the primary key associated with the token.
# ``database_id`` is ``None`` during the "Create database" wizard — the
# callback caches the access token in the KV store instead of inserting
# a row in ``database_user_oauth2_tokens``.
"database_id": database.id,
"user_id": g.user.id,
# In multi-instance deployments there might be a single proxy handling
@@ -716,25 +710,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
if oauth2_config is None:
raise OAuth2Error("No configuration found for OAuth2")
# Pre-create flow: the database has no row yet, so we can't look it up by id
# in the callback. Stash the engine spec + oauth2 config in the KV store
# alongside the existing PKCE entry, keyed by the same ``tab_id``. The
# callback reads this to exchange the code without a persisted ``Database``.
if database.id is None:
state["engine"] = cls.engine
KeyValueDAO.delete_expired_entries(KeyValueResource.OAUTH2_PRE_CREATE_TOKEN)
KeyValueDAO.create_entry(
resource=KeyValueResource.OAUTH2_PRE_CREATE_TOKEN,
value={
"engine": cls.engine,
"config": oauth2_config,
},
codec=JsonKeyValueCodec(),
key=UUID(tab_id),
expires_on=datetime.now() + timedelta(minutes=5),
)
db.session.commit()
oauth_url = cls.get_oauth2_authorization_uri(
oauth2_config,
state,
+40 -2
View File
@@ -21,10 +21,11 @@ import logging
import re
from datetime import datetime
from re import Pattern
from typing import Any, Optional, TYPE_CHECKING
from typing import Any, Callable, Optional, TYPE_CHECKING
from flask_babel import gettext as __
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, JSON
from sqlalchemy import types
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, INTERVAL, JSON
from sqlalchemy.dialects.postgresql.base import PGInspector
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
@@ -135,6 +136,34 @@ def parse_options(connect_args: dict[str, Any]) -> dict[str, str]:
return {token[0]: token[1] for token in tokens}
def _normalize_interval(v: Any) -> Optional[float]:
"""Convert PostgreSQL INTERVAL values to milliseconds.
psycopg2 and psycopg3 always return INTERVAL values as datetime.timedelta
objects. We convert to milliseconds so users can apply the built-in
"DURATION" number format for human-readable display (e.g.,
"1d 2h 30m 45s") and so the values participate cleanly in numeric
aggregations in bar/pie charts.
Returns None for the NULL case (preserves NULL semantics) and for any
unexpected non-timedelta type (avoids producing a mixed-type column
when an unfamiliar driver surfaces something other than timedelta).
"""
if v is None:
return None
if hasattr(v, "total_seconds"):
return v.total_seconds() * 1000
# Defensive: psycopg2/3 should always hand us a timedelta. If a future
# driver doesn't, surface the surprise in the logs rather than silently
# dropping the value so operators can diagnose it.
logger.warning(
"Cannot normalize PostgreSQL INTERVAL value of type %s to numeric; "
"returning None.",
type(v).__name__,
)
return None
class PostgresBaseEngineSpec(BaseEngineSpec):
"""Abstract class for Postgres 'like' databases"""
@@ -526,8 +555,17 @@ class PostgresEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec):
ENUM(),
GenericDataType.STRING,
),
(
re.compile(r"^interval", re.IGNORECASE),
INTERVAL(),
GenericDataType.NUMERIC,
),
)
column_type_mutators: dict[types.TypeEngine, Callable[[Any], Any]] = {
INTERVAL: _normalize_interval,
}
@classmethod
def get_schema_from_engine_params(
cls,
-1
View File
@@ -46,7 +46,6 @@ class KeyValueResource(StrEnum):
METASTORE_CACHE = "superset_metastore_cache"
LOCK = "lock"
PKCE_CODE_VERIFIER = "pkce_code_verifier"
OAUTH2_PRE_CREATE_TOKEN = "oauth2_pre_create_token" # noqa: S105
SQLLAB_PERMALINK = "sqllab_permalink"
+13 -1
View File
@@ -53,7 +53,11 @@ from superset_core.queries.models import (
)
from superset import security_manager
from superset.exceptions import SupersetParseError, SupersetSecurityException
from superset.exceptions import (
SupersetException,
SupersetParseError,
SupersetSecurityException,
)
from superset.explorables.base import TimeGrainDict
from superset.jinja_context import BaseTemplateProcessor, get_template_processor
from superset.models.helpers import (
@@ -99,6 +103,14 @@ class SqlTablesMixin: # pylint: disable=too-few-public-methods
)
except (SupersetSecurityException, SupersetParseError, TemplateError):
return []
except SupersetException as ex:
# Jinja macros such as ``{{ dataset(id) }}`` or ``{{ metric(...) }}``
# may reference resources that no longer exist (e.g. a deleted
# dataset). Surfacing the failure here would break list endpoints
# that include ``sql_tables`` in their payload, hiding every saved
# query from the user. Treat it as a parse failure instead.
logger.warning("Unable to extract tables from SQL via Jinja: %s", ex)
return []
class Query(
+1 -9
View File
@@ -402,15 +402,7 @@ class OAuth2State(TypedDict, total=False):
Type for the state passed during OAuth2.
"""
# ``database_id`` is ``None`` during the "Create database" wizard, where the
# OAuth2 dance runs before the database has been persisted. In that case the
# access token is cached in the KV store keyed by ``tab_id`` until the user
# saves the database.
database_id: int | None
database_id: int
user_id: int
default_redirect_uri: str
tab_id: str
# Engine backend code (e.g. ``"semanticapi"``), present only for pre-create
# dances so the callback can resolve the engine spec without a persisted
# database row.
engine: str
+5 -57
View File
@@ -24,11 +24,10 @@ import secrets
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Any, Iterator, TYPE_CHECKING
from uuid import UUID
import backoff
import jwt
from flask import current_app as app, g, url_for
from flask import current_app as app, url_for
from marshmallow import EXCLUDE, fields, post_load, Schema, validate
from werkzeug.routing import BuildError
@@ -88,7 +87,7 @@ def generate_code_challenge(code_verifier: str) -> str:
)
def get_oauth2_access_token(
config: OAuth2ClientConfig,
database_id: int | None,
database_id: int,
user_id: int,
db_engine_spec: type[BaseEngineSpec],
) -> str | None:
@@ -101,18 +100,10 @@ def get_oauth2_access_token(
simultaneous requests for refreshing a stale token; in that case only the first
process to acquire the lock will perform the refresh, and othe process should find a
a valid token when they retry.
When ``database_id`` is ``None`` (the "Create database" wizard, where the
database hasn't been persisted yet), look up the token in the KV store under
``g.oauth2_tab_id``. The token was cached there by
:class:`OAuth2StoreTokenCommand` after the wizard's OAuth2 dance.
""" # noqa: E501
# pylint: disable=import-outside-toplevel
from superset.models.core import DatabaseUserOAuth2Tokens
if not database_id:
return _get_pre_create_access_token(user_id)
token = (
db.session.query(DatabaseUserOAuth2Tokens)
.filter_by(user_id=user_id, database_id=database_id)
@@ -133,39 +124,6 @@ def get_oauth2_access_token(
return None
def _get_pre_create_access_token(user_id: int) -> str | None:
"""
Look up a pre-create OAuth2 token in the KV store.
The KV entry is keyed by the wizard's ``tab_id``; the frontend passes it on
every "Test Connection" request, and the test-connection command exposes it
via :data:`flask.g.oauth2_tab_id`.
"""
# pylint: disable=import-outside-toplevel
from superset.daos.key_value import KeyValueDAO
from superset.key_value.types import JsonKeyValueCodec, KeyValueResource
tab_id = getattr(g, "oauth2_tab_id", None)
if not tab_id:
return None
try:
tab_uuid = UUID(tab_id)
except (TypeError, ValueError):
return None
cached = KeyValueDAO.get_value(
resource=KeyValueResource.OAUTH2_PRE_CREATE_TOKEN,
key=tab_uuid,
codec=JsonKeyValueCodec(),
)
if not cached:
return None
if cached.get("user_id") != user_id:
# the tab_id is the user's own, but be defensive
return None
return cached.get("access_token")
def refresh_oauth2_token(
config: OAuth2ClientConfig,
database_id: int,
@@ -245,20 +203,14 @@ def refresh_oauth2_token(
def encode_oauth2_state(state: OAuth2State) -> str:
"""
Encode the OAuth2 state.
``database_id`` is ``None`` for the pre-create-database OAuth2 dance,
which means the token won't be persisted to ``database_user_oauth2_tokens``
when the callback runs see :class:`OAuth2StoreTokenCommand`.
"""
payload: dict[str, Any] = {
"exp": datetime.now(tz=timezone.utc) + JWT_EXPIRATION,
"database_id": state.get("database_id"),
"database_id": state["database_id"],
"user_id": state["user_id"],
"default_redirect_uri": state["default_redirect_uri"],
"tab_id": state["tab_id"],
}
if engine := state.get("engine"):
payload["engine"] = engine
encoded_state = jwt.encode(
payload=payload,
@@ -273,11 +225,10 @@ def encode_oauth2_state(state: OAuth2State) -> str:
class OAuth2StateSchema(Schema):
database_id = fields.Int(required=True, allow_none=True)
database_id = fields.Int(required=True)
user_id = fields.Int(required=True)
default_redirect_uri = fields.Str(required=True)
tab_id = fields.Str(required=True)
engine = fields.Str(required=False, load_default=None)
# pylint: disable=unused-argument
@post_load
@@ -286,15 +237,12 @@ class OAuth2StateSchema(Schema):
data: dict[str, Any],
**kwargs: Any,
) -> OAuth2State:
state: OAuth2State = {
return {
"database_id": data["database_id"],
"user_id": data["user_id"],
"default_redirect_uri": data["default_redirect_uri"],
"tab_id": data["tab_id"],
}
if data.get("engine"):
state["engine"] = data["engine"]
return state
class Meta: # pylint: disable=too-few-public-methods
# ignore `exp`
+50
View File
@@ -152,3 +152,53 @@ def test_time_grain_validation_with_config_addons(app_context: None) -> None:
}
result = schema.load(custom_data)
assert result["time_grain"] == "PT10M"
def test_post_processing_operation_validates_options(app_context: None) -> None:
"""options are validated against the operation's option schema (leniently)."""
from superset.charts.schemas import ChartDataPostProcessingOperationSchema
schema = ChartDataPostProcessingOperationSchema()
# Valid prophet options load.
schema.load(
{
"operation": "prophet",
"options": {
"time_grain": "P1D",
"periods": 7,
"confidence_interval": 0.8,
},
}
)
# Out-of-range confidence_interval (must be 0-1) on a declared field is
# rejected.
with pytest.raises(ValidationError) as exc_info:
schema.load(
{
"operation": "prophet",
"options": {
"time_grain": "P1D",
"periods": 7,
"confidence_interval": 2.0,
},
}
)
assert "options" in exc_info.value.messages
# Extra/unknown keys are tolerated (lenient validation).
schema.load(
{
"operation": "prophet",
"options": {
"time_grain": "P1D",
"periods": 7,
"confidence_interval": 0.8,
"some_future_option": True,
},
}
)
# An operation without a dedicated schema accepts arbitrary options.
schema.load({"operation": "flatten", "options": {"anything": [1, 2, 3]}})
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, cast
from typing import Any
from unittest.mock import MagicMock
import pytest
@@ -27,7 +27,6 @@ from superset.daos.database import DatabaseUserOAuth2TokensDAO
from superset.databases.schemas import OAuth2ProviderResponseSchema
from superset.exceptions import OAuth2Error
from superset.models.core import Database
from superset.superset_typing import OAuth2State
from superset.utils.oauth2 import decode_oauth2_state, encode_oauth2_state
@@ -136,80 +135,6 @@ def test_run_success(
mock_create.assert_called_once()
def test_run_pre_create_caches_token_in_kv(mocker: MockerFixture) -> None:
"""
With ``state.database_id is None`` the command reads the engine + config
from the pre-create KV entry, exchanges the code, and writes the token
back to KV *not* to ``database_user_oauth2_tokens``.
"""
state: OAuth2State = {
"user_id": 1,
"database_id": None,
"default_redirect_uri": "http://localhost:8088/api/v1/oauth2/",
"tab_id": "3a3a3a3a-3a3a-3a3a-3a3a-3a3a3a3a3a3a",
"engine": "semanticapi",
}
parameters: dict[str, Any] = {
"code": "the-code",
"state": encode_oauth2_state(state),
}
kv_dao = mocker.patch("superset.commands.database.oauth2.KeyValueDAO")
kv_dao.get_value.side_effect = [
{"engine": "semanticapi", "config": {"id": "x", "secret": "y"}}, # validate()
None, # code_verifier lookup
]
engine_spec = mocker.MagicMock()
engine_spec.engine = "semanticapi"
engine_spec.get_oauth2_token.return_value = {
"access_token": "fresh-token",
"expires_in": 3600,
"refresh_token": "refresh",
}
mocker.patch(
"superset.commands.database.oauth2.get_engine_spec",
return_value=engine_spec,
)
dao_get_db = mocker.patch.object(DatabaseUserOAuth2TokensDAO, "get_database")
dao_create = mocker.patch.object(DatabaseUserOAuth2TokensDAO, "create")
result = OAuth2StoreTokenCommand(
cast(OAuth2ProviderResponseSchema, parameters),
).run()
assert result is None
dao_get_db.assert_not_called()
dao_create.assert_not_called()
kv_dao.upsert_entry.assert_called_once()
upsert_kwargs = kv_dao.upsert_entry.call_args.kwargs
assert upsert_kwargs["value"]["access_token"] == "fresh-token" # noqa: S105
assert upsert_kwargs["value"]["user_id"] == 1
def test_run_pre_create_missing_kv_entry(mocker: MockerFixture) -> None:
"""
Pre-create flow with no cached entry should fail with a clear OAuth2Error.
"""
state: OAuth2State = {
"user_id": 1,
"database_id": None,
"default_redirect_uri": "http://localhost:8088/api/v1/oauth2/",
"tab_id": "3a3a3a3a-3a3a-3a3a-3a3a-3a3a3a3a3a3a",
}
parameters: dict[str, Any] = {"code": "x", "state": encode_oauth2_state(state)}
kv_dao = mocker.patch("superset.commands.database.oauth2.KeyValueDAO")
kv_dao.get_value.return_value = None
with pytest.raises(OAuth2Error) as exc_info:
OAuth2StoreTokenCommand(
cast(OAuth2ProviderResponseSchema, parameters),
).validate()
assert "Pre-create OAuth2 context" in exc_info.value.error.extra["error"]
def test_run_existing_token(
mocker: MockerFixture,
mock_database: MagicMock,
@@ -238,6 +238,136 @@ def test_update_native_filter_config_default_rootpath_preserved():
assert scope["excluded"] == []
def test_update_id_refs_remaps_charts_in_scope():
"""
Regression for #26338: ``chartsInScope`` on a native filter holds chart
IDs and must be remapped from source-env IDs to destination-env IDs
during import.
The export side already converts ``chartsInScope`` IDs to UUIDs (see
``export_example.py:325``). The import side must symmetrically convert
them back to the destination environment's chart IDs. Without that
remap, the field carries stale source IDs into the imported dashboard
and breaks ``filtersInScope`` / ``filtersOutScope`` computation
filters end up applied to the wrong charts (or none at all).
"""
from superset.commands.dashboard.importers.v1.utils import update_id_refs
config: dict[str, Any] = {
"position": {
"CHART1": {
"id": "CHART1",
"meta": {"chartId": 101, "uuid": "uuid1"},
"type": "CHART",
},
"CHART2": {
"id": "CHART2",
"meta": {"chartId": 102, "uuid": "uuid2"},
"type": "CHART",
},
},
"metadata": {
"native_filter_configuration": [
{
"id": "NATIVE_FILTER-region",
"scope": {"rootPath": ["ROOT_ID"], "excluded": []},
# chartsInScope contains source-env chart IDs.
"chartsInScope": [101, 102, 103],
}
],
},
}
chart_ids = {"uuid1": 1, "uuid2": 2}
dataset_info: dict[str, dict[str, Any]] = {}
fixed = update_id_refs(config, chart_ids, dataset_info)
filter_config = fixed["metadata"]["native_filter_configuration"][0]
# Resolved IDs are remapped; unknown IDs (103) are dropped rather than
# left to silently bind to whatever chart owns that integer in the
# destination environment.
assert filter_config["chartsInScope"] == [1, 2]
def test_update_id_refs_remaps_cross_filter_charts_in_scope():
"""
Companion to test_update_id_refs_remaps_charts_in_scope. Cross-filter
config also stores ``chartsInScope`` (under ``crossFilters`` per chart)
and must be remapped on import for the same reason.
"""
from superset.commands.dashboard.importers.v1.utils import update_id_refs
config: dict[str, Any] = {
"position": {
"CHART1": {
"id": "CHART1",
"meta": {"chartId": 101, "uuid": "uuid1"},
"type": "CHART",
},
"CHART2": {
"id": "CHART2",
"meta": {"chartId": 102, "uuid": "uuid2"},
"type": "CHART",
},
},
"metadata": {
"chart_configuration": {
"101": {
"id": 101,
"crossFilters": {
"scope": {"rootPath": ["ROOT_ID"], "excluded": []},
"chartsInScope": [101, 102, 103],
},
},
},
},
}
chart_ids = {"uuid1": 1, "uuid2": 2}
dataset_info: dict[str, dict[str, Any]] = {}
fixed = update_id_refs(config, chart_ids, dataset_info)
cross_filters = fixed["metadata"]["chart_configuration"]["1"]["crossFilters"]
assert cross_filters["chartsInScope"] == [1, 2]
def test_update_id_refs_remaps_global_chart_configuration_charts_in_scope():
"""
Per-chart and native-filter ``chartsInScope`` are remapped by their own
branches; ``global_chart_configuration.chartsInScope`` lives next to
``global_chart_configuration.scope.excluded`` and needs the same treatment
so the global cross-filter scope cache doesn't keep stale source-env IDs.
"""
from superset.commands.dashboard.importers.v1.utils import update_id_refs
config: dict[str, Any] = {
"position": {
"CHART1": {
"id": "CHART1",
"meta": {"chartId": 101, "uuid": "uuid1"},
"type": "CHART",
},
"CHART2": {
"id": "CHART2",
"meta": {"chartId": 102, "uuid": "uuid2"},
"type": "CHART",
},
},
"metadata": {
"global_chart_configuration": {
"scope": {"rootPath": ["ROOT_ID"], "excluded": []},
"chartsInScope": [101, 102, 103],
},
},
}
chart_ids = {"uuid1": 1, "uuid2": 2}
dataset_info: dict[str, dict[str, Any]] = {}
fixed = update_id_refs(config, chart_ids, dataset_info)
assert fixed["metadata"]["global_chart_configuration"]["chartsInScope"] == [1, 2]
def test_update_id_refs_cross_filter_chart_configuration_key_and_excluded_mapping():
from superset.commands.dashboard.importers.v1.utils import update_id_refs
@@ -15,14 +15,14 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from datetime import datetime, timedelta
from typing import Any, Optional
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from sqlalchemy import column, types
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, JSON
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, INTERVAL, JSON
from sqlalchemy.engine.interfaces import Dialect
from sqlalchemy.engine.url import make_url
@@ -87,6 +87,8 @@ def test_convert_dttm(
("TIME", types.Time, None, GenericDataType.TEMPORAL, True),
# Boolean
("BOOLEAN", types.Boolean, None, GenericDataType.BOOLEAN, False),
# Interval (mapped to NUMERIC for chart rendering)
("INTERVAL", INTERVAL, None, GenericDataType.NUMERIC, False),
],
)
def test_get_column_spec(
@@ -366,3 +368,38 @@ class TestRedshiftDetection:
spec.update_params_from_encrypted_extra(database, params)
assert "pool_events" not in params
def test_interval_type_mutator() -> None:
"""
DB Eng Specs (postgres): Test INTERVAL type mutator
INTERVAL values are converted to milliseconds so users can apply
the built-in "DURATION" number format for human-readable display.
"""
mutator = spec.column_type_mutators[INTERVAL]
# Timedelta conversion — the only path psycopg2/psycopg3 actually
# exercises. Result is in milliseconds for compatibility with the
# DURATION formatter.
td = timedelta(days=1, hours=2, minutes=30, seconds=45)
assert mutator(td) == 95445000.0 # (1*86400 + 2*3600 + 30*60 + 45) * 1000
# Zero duration
assert mutator(timedelta(0)) == 0.0
# Negative interval
assert mutator(timedelta(days=-1)) == -86400000.0
# None preserves NULL semantics (not converted to 0)
assert mutator(None) is None
# Unexpected non-timedelta types fall through to the defensive
# `return None` (and emit a warning) rather than producing a
# mixed-type column.
assert mutator("1 day 02:30:45") is None
assert mutator("P1DT2H30M45S") is None
assert mutator(12345) is None
assert mutator(True) is None
assert mutator([1, 2, 3]) is None
assert mutator({"days": 1}) is None
+45 -12
View File
@@ -21,8 +21,14 @@ from flask_appbuilder import Model
from jinja2.exceptions import TemplateError
from pytest_mock import MockerFixture
from superset.commands.dataset.exceptions import DatasetNotFoundError
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import SupersetParseError, SupersetSecurityException
from superset.exceptions import (
SupersetParseError,
SupersetSecurityException,
SupersetTemplateException,
)
from superset.models import sql_lab as sql_lab_module
from superset.models.sql_lab import Query, SavedQuery
@@ -34,34 +40,61 @@ from superset.models.sql_lab import Query, SavedQuery
],
)
@pytest.mark.parametrize(
"exception",
("exception", "should_warn"),
[
SupersetSecurityException(
SupersetError(
error_type=SupersetErrorType.QUERY_SECURITY_ACCESS_ERROR,
message="",
level=ErrorLevel.ERROR,
)
# Original silent handler — security/parse/template errors are
# expected during list rendering and produce no log noise.
(
SupersetSecurityException(
SupersetError(
error_type=SupersetErrorType.QUERY_SECURITY_ACCESS_ERROR,
message="",
level=ErrorLevel.ERROR,
)
),
False,
),
SupersetParseError(
sql="INVALID SQL",
message="Invalid SQL syntax",
(
SupersetParseError(
sql="INVALID SQL",
message="Invalid SQL syntax",
),
False,
),
TemplateError,
(TemplateError, False),
# ``{{ dataset(id) }}`` referencing a deleted dataset previously
# bubbled up through ``sql_tables`` and broke saved-query list
# endpoints (see issue #32771). The new handler swallows it but
# logs a warning so the underlying breakage is still observable —
# pinned here so a future refactor that collapses the case into
# the silent handler fails this test.
(DatasetNotFoundError("Dataset 1 not found!"), True),
(SupersetTemplateException("Template rendering failed"), True),
],
)
def test_sql_tables_mixin_sql_tables_exception(
klass: type[Model],
exception: Exception,
should_warn: bool,
mocker: MockerFixture,
) -> None:
mocker.patch(
"superset.models.sql_lab.process_jinja_sql",
side_effect=exception,
)
warning_spy = mocker.spy(sql_lab_module.logger, "warning")
assert klass(sql="SELECT 1", database=MagicMock()).sql_tables == []
if should_warn:
assert warning_spy.call_count == 1, (
f"{type(exception).__name__} should hit the warning-logging "
"handler; if this fails, the case was likely collapsed into "
"the silent first-handler clause."
)
else:
warning_spy.assert_not_called()
@pytest.mark.parametrize(
"klass",
@@ -0,0 +1,58 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations
import pyarrow as pa
from superset_core.semantic_layers.types import AggregationType, Metric
def test_metric_aggregation_defaults_to_none() -> None:
metric = Metric(
id="x",
name="x",
type=pa.float64(),
definition="sum(x)",
)
assert metric.aggregation is None
def test_metric_accepts_aggregation_type() -> None:
metric = Metric(
id="x",
name="x",
type=pa.float64(),
definition="sum(x)",
aggregation=AggregationType.SUM,
)
assert metric.aggregation is AggregationType.SUM
def test_aggregation_type_is_string_enum() -> None:
# Behaves as a string for equality and serialization, so it can be sent
# over JSON without an explicit converter.
assert AggregationType.SUM == "SUM"
assert AggregationType.COUNT_DISTINCT.value == "COUNT_DISTINCT"
assert {a.value for a in AggregationType} == {
"SUM",
"COUNT",
"MIN",
"MAX",
"AVG",
"COUNT_DISTINCT",
"OTHER",
}
-77
View File
@@ -458,83 +458,6 @@ def test_encode_decode_oauth2_state(
assert decoded["user_id"] == 2
def test_encode_decode_oauth2_state_pre_create(mocker: MockerFixture) -> None:
"""
The pre-create dance encodes ``database_id=None`` and an ``engine``
field; both must survive the round-trip so the callback can resolve the
engine spec without a persisted database.
"""
from superset.superset_typing import OAuth2State
mocker.patch(
"flask.current_app.config",
{"SECRET_KEY": "test-secret-key", "DATABASE_OAUTH2_JWT_ALGORITHM": "HS256"},
)
state: OAuth2State = {
"database_id": None,
"user_id": 2,
"default_redirect_uri": "http://localhost:8088/api/v1/oauth2/",
"tab_id": "abc",
"engine": "semanticapi",
}
with freeze_time("2024-01-01"):
decoded = decode_oauth2_state(encode_oauth2_state(state))
assert decoded["database_id"] is None
assert decoded["engine"] == "semanticapi"
def test_get_oauth2_access_token_pre_create_hit(mocker: MockerFixture) -> None:
"""
With ``database_id=None`` and a matching KV entry, the cached pre-create
token is returned.
"""
mocker.patch(
"superset.utils.oauth2.g",
oauth2_tab_id="3a3a3a3a-3a3a-3a3a-3a3a-3a3a3a3a3a3a",
)
kv_dao = mocker.patch("superset.daos.key_value.KeyValueDAO")
kv_dao.get_value.return_value = {
"access_token": "cached-token",
"user_id": 7,
}
assert (
get_oauth2_access_token(DUMMY_OAUTH2_CONFIG, None, 7, mocker.MagicMock())
== "cached-token"
)
def test_get_oauth2_access_token_pre_create_miss(mocker: MockerFixture) -> None:
"""
Without a tab id (or with no KV entry / wrong user) the lookup returns None.
"""
mocker.patch("superset.utils.oauth2.g", oauth2_tab_id=None)
assert (
get_oauth2_access_token(DUMMY_OAUTH2_CONFIG, None, 7, mocker.MagicMock())
is None
)
mocker.patch(
"superset.utils.oauth2.g",
oauth2_tab_id="3a3a3a3a-3a3a-3a3a-3a3a-3a3a3a3a3a3a",
)
kv_dao = mocker.patch("superset.daos.key_value.KeyValueDAO")
kv_dao.get_value.return_value = None
assert (
get_oauth2_access_token(DUMMY_OAUTH2_CONFIG, None, 7, mocker.MagicMock())
is None
)
kv_dao.get_value.return_value = {"access_token": "x", "user_id": 99}
# token belongs to another user
assert (
get_oauth2_access_token(DUMMY_OAUTH2_CONFIG, None, 7, mocker.MagicMock())
is None
)
def test_get_oauth2_access_token_lock_not_acquired_no_error_log(
mocker: MockerFixture,
caplog: pytest.LogCaptureFixture,