Compare commits

..
Author SHA1 Message Date
Enzo MartellucciandClaude Sonnet 5 5a6c1b977b refactor(sqllab): drop the redundant canSaveDataset prop from SaveQuery
SaveQuery already receives the query's result columns and computes its
own SQL-staleness check, so master's canSaveDataset prop (threaded
through SqlEditor -> SaveQuery -> SaveDatasetActionButton) duplicated
the same "did the query succeed" condition. Fold it into the single
check SaveQuery already owns instead of ANDing two independently
computed booleans together.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-21 14:17:12 +02:00
Enzo Martellucci e8577368d3 Merge branch 'master' into enxdev/fix/sql-lab-save-dataset
# Conflicts:
#	superset-frontend/src/SqlLab/components/SaveDatasetActionButton/SaveDatasetActionButton.test.tsx
#	superset-frontend/src/SqlLab/components/SaveDatasetActionButton/index.tsx
#	superset-frontend/src/SqlLab/components/SaveQuery/index.tsx
2026-08-21 14:15:55 +02:00
Enzo Martellucci 540f8cb2d0 fix(sqllab): invalidate the dataset-save gate when the editor SQL changes 2026-08-19 16:52:40 +02:00
Enzo Martellucci aae997e546 fix(sql_lab): return 400 not 500 when raise_for_access hits malformed Jinja 2026-08-19 16:02:09 +02:00
65 changed files with 386 additions and 2130 deletions
+4 -4
View File
@@ -78,14 +78,14 @@ jobs:
USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true' || 'false' }}
services:
postgres:
image: ghcr.io/apache/superset/ci/postgres:17-alpine
image: postgres:17-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
- 15432:5432
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
@@ -186,14 +186,14 @@ jobs:
GITHUB_TOKEN: ${{ github.token }}
services:
postgres:
image: ghcr.io/apache/superset/ci/postgres:17-alpine
image: postgres:17-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
- 15432:5432
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
@@ -53,7 +53,9 @@ jobs:
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: ghcr.io/apache/superset/ci/mysql:8.0
image: mysql:8.0
# Authenticated pulls use our higher Docker Hub rate limit. Empty on
# fork PRs (secrets unavailable) -> runner falls back to anonymous.
env:
MYSQL_ROOT_PASSWORD: root
ports:
@@ -64,7 +66,7 @@ jobs:
--health-timeout=5s
--health-retries=5
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
@@ -141,7 +143,7 @@ jobs:
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: ghcr.io/apache/superset/ci/postgres:17-alpine
image: postgres:17-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -150,7 +152,7 @@ jobs:
# GitHub action runner's default installations
- 15432:5432
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
@@ -200,7 +202,7 @@ jobs:
sqlite:///${{ github.workspace }}/.temp/examples.db?check_same_thread=true
services:
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
@@ -52,7 +52,7 @@ jobs:
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
services:
postgres:
image: ghcr.io/apache/superset/ci/postgres:17-alpine
image: postgres:17-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -61,7 +61,7 @@ jobs:
# GitHub action runner's default installations
- 15432:5432
presto:
image: ghcr.io/apache/superset/ci/presto:350-e.6
image: starburstdata/presto:350-e.6
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -70,7 +70,7 @@ jobs:
# GitHub action runner's default installations
- 15433:8080
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
@@ -116,7 +116,7 @@ jobs:
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: ghcr.io/apache/superset/ci/postgres:17-alpine
image: postgres:17-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -125,7 +125,7 @@ jobs:
# GitHub action runner's default installations
- 15432:5432
redis:
image: ghcr.io/apache/superset/ci/redis:7-alpine
image: redis:7-alpine
ports:
- 16379:6379
steps:
+1 -1
View File
@@ -304,7 +304,7 @@ pre-commit run eslint # Frontend linting
## Platform-Specific Instructions
- **[CLAUDE.md](CLAUDE.md)** - For Claude/Anthropic tools
- **[.github/copilot-instructions.md](.github/copilot-instructions.md)** - For GitHub Copilot
- **[.github/copilot-instructions.md](.github/copilot-instructions.md)** - For GitHub Copilot
- **[GEMINI.md](GEMINI.md)** - For Google Gemini tools
- **[GPT.md](GPT.md)** - For OpenAI/ChatGPT tools
- **[.cursor/rules/dev-standard.mdc](.cursor/rules/dev-standard.mdc)** - For Cursor editor
-17
View File
@@ -125,23 +125,6 @@ dialect; each package's constraint in `pyproject.toml` documents why.
No application-level configuration changes are required for deployments
that don't touch SQLAlchemy directly.
### New metric aggregates: MEDIAN, Sample Standard Deviation, Sample Variance
`MEDIAN`, `STDDEV_SAMP`, and `VAR_SAMP` are now available anywhere a metric
aggregate is chosen (every chart type, SQL Lab, MCP), not only in Pivot
Table's controls. Support is opt-in per database engine *spec class*,
verified against a live instance before being enabled: Postgres, MySQL
(`STDDEV_SAMP`/`VAR_SAMP` only, no `MEDIAN`), DuckDB, and Redshift (inherits
Postgres's support, not yet separately verified) ship enabled in this
release. Engine specs that subclass one of those (e.g. MariaDB, Aurora
MySQL/Postgres, TimescaleDB) inherit the same support, on the same
not-yet-independently-verified basis. Picking one of these aggregates on a
database that has not opted in returns a clear "not supported on this
database" error rather than a failed query. See
`docs/sip/median-stddev-variance-aggregates.md` for the full design
rationale, including why this is safe to add without reintroducing the
totals/subtotals correctness bug fixed by #41184 (SIP-216).
### Soft delete is on by default, and purging is live
`SOFT_DELETE` now ships **on** (`DEFAULT_FEATURE_FLAGS`), so deleting a
@@ -1,236 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# SIP: System-wide MEDIAN, Sample Standard Deviation, and Sample Variance metric aggregates
## [DRAFT — proposal for discussion]
This document now has an accompanying implementation in this PR, for the
proposed mechanism plus a first, empirically-verified engine set (Postgres,
MySQL, DuckDB, Redshift by inheritance). It is intended to seed discussion on
whether this is the right shape and scope before it goes up for a formal SIP
vote, not to pre-empt that discussion, the code exists so reviewers have a
concrete design to react to rather than a description of one.
## Motivation
Before #41184 (SIP-216, the non-additive-totals fix), the Pivot Table chart
exposed an "Aggregation function" control with 18 choices, including
`Median`, `Sample Standard Deviation`, `Sample Variance`, `First`, `Last`,
`Count Unique Values`, and `List Unique Values`. #41184 deleted that control
wholesale, and deliberately so: it re-aggregated already-aggregated cell
values to compute totals/subtotals, which is exactly the class of bug
SIP-216 fixed (summing per-group averages, averaging per-group medians, etc.
produces silently wrong totals). #42761 subsequently restored the one piece
of that control's functionality that was cleanly separable from the
correctness bug, the "show as % of row/column/total" display option,
redesigned as a decoupled, post-hoc-only `showValuesAs` control.
A user has since noticed that several of the other pre-#41184 options never
came back. Checking today's metric aggregate list (`AVG, COUNT,
COUNT_DISTINCT, MAX, MIN, SUM`, see
`superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`),
most of these have a reasonable equivalent already: `Count Unique Values`
maps to `COUNT_DISTINCT`; `Count`/`Average`/`Max`/`Min` are already standard
aggregates; the two "fraction of" variants are already covered by
`showValuesAs`. But `Median`, `Sample Standard Deviation`, and `Sample
Variance` have no equivalent today anywhere in Superset, not just in Pivot
Table, in any chart type, since the aggregate list is shared across the
whole app.
This is a real, currently-live gap, not a hypothetical one:
`superset/mcp_service/chart/chart_utils.py`, `schemas.py`, and
`prompts/create_chart_guided.py` already treat `STDDEV`, `VAR`, `MEDIAN`,
and `PERCENTILE` as valid aggregate values in their own validation and
documentation, but those values are never recognized by
`superset/connectors/sqla/models.py`'s `sqla_aggregations` dict (the actual
mapping from aggregate name to SQL), so an AI agent using the MCP tool to
build a chart with `"aggregate": "STDDEV"` today creates a chart that
**errors at query time** with "Adhoc metric aggregate is invalid." This SIP
proposes closing that gap for real, at the source, rather than patching
around it in MCP.
## Proposed change
Add `MEDIAN`, `STDDEV_SAMP`, and `VAR_SAMP` as first-class, system-wide
metric aggregates, available anywhere a metric aggregate is chosen (every
chart type, SQL Lab metric picker, MCP), not as a Pivot-Table-specific
control.
**Why this is safe with respect to SIP-216, and needs no Pivot-Table-specific
code at all:** Pivot Table's non-additive-totals machinery
(`superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/utilities.ts`)
already classifies any metric aggregate not in `ADDITIVE_AGGREGATES = {SUM,
COUNT, MIN, MAX}` as non-additive, which routes totals/subtotals through the
correct DB-`GROUPING SETS`-rollup path rather than client-side
re-aggregation (`AVG` and `COUNT_DISTINCT` already go through this path
today). `MEDIAN`/`STDDEV_SAMP`/`VAR_SAMP` fall into that bucket
automatically, with zero changes needed to the additivity logic. So once
these are valid, buildable SQL aggregates, Pivot Table (and every other
chart) gets correct behavior for free. This is the version of "restore the
control" that does not reopen the bug that was just fixed.
**Where the actual change needs to land, and what this PR does:**
1. **Done.** `superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`,
extended the `Aggregate` type.
2. **Done.** `superset-frontend/src/explore/constants.ts`, added to `AGGREGATES`
(drives `AGGREGATES_OPTIONS`, the dropdown in `AdhocMetricEditPopover`).
3. **Done**, but not consolidated. `superset/connectors/sqla/models.py`
(`sqla_aggregations`) and `superset/models/helpers.py`
(`ExploreMixin.sqla_aggregations`) are both wired to consult the new
`BaseEngineSpec.get_extended_aggregation_func`, in addition to their
existing 6-aggregate dict, so neither's original, already-tested behavior
changed. They remain two separate dicts, consolidating them into one
source of truth is left as a follow-up (see Open questions).
4. **Done**, and it surfaced a second, smaller bug on top of the one this SIP
opened with: MCP's own aggregate names (`STDDEV`, `VAR`) never matched any
real Superset aggregate, before or after this PR, they were always going
to error regardless of what this SIP does. `superset/mcp_service/chart/*`
now accepts the old shorthand as an alias, normalized to the real,
unambiguous names (`STDDEV_SAMP`, `VAR_SAMP`) this PR ships, and the guided
prompt text points at the correct names going forward. `MEDIAN`/
`PERCENTILE` were already spelled correctly in MCP; `PERCENTILE` remains
unimplemented (it needs a parameter this schema has no field for) and is
unchanged by this PR, out of scope here.
**The part that needs real engineering care, this must not be a blind
`sa.func.MEDIAN` / `sa.func.STDDEV_SAMP` / `sa.func.VAR_SAMP`:**
`sqla_aggregations` today is a flat, engine-unaware dict (`sa.func.AVG`,
etc., SQLAlchemy emits whatever function name it is given, with zero
validation that the target dialect actually has it). Superset already has
precedent for exactly this class of per-engine capability difference:
`BaseEngineSpec.supports_grouping_sets` and `_time_grain_expressions`, both
introduced by #41184 itself. This SIP proposes the same shape, a new
per-engine-overridable mechanism (for example
`BaseEngineSpec.get_aggregate_sql(aggregate, column)` with a sensible
default, overridden per engine spec where the default does not hold),
rather than a single hardcoded dict.
Verified findings so far (via `sqlglot.transpile`, cross-checked against
known engine docs; **not** exhaustively tested against live databases, that
is necessary follow-up work this SIP alone cannot complete):
| Engine | `MEDIAN(x)` | `STDDEV_SAMP(x)` | `VAR_SAMP(x)` |
|---|---|---|---|
| Postgres | `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY x)` | native | native |
| MySQL | no native equivalent, needs explicit "unsupported" handling, not a blind emit | native | MySQL's `VARIANCE()` is an alias for `VAR_POP` (population), not `VAR_SAMP` (sample); a naive dialect-name substitution would silently compute the wrong statistic and needs an explicit, verified expression instead |
| SQLite | only if the specific build was compiled with the (non-default) `SQLITE_ENABLE_PERCENTILE` extension (added in SQLite 3.43, 2023), cannot be assumed available | not available in core SQLite | not available in core SQLite |
| BigQuery / Snowflake / DuckDB / Redshift / Oracle / T-SQL / Databricks / Spark | native `MEDIAN(x)` | native | native on BigQuery/Snowflake/Databricks/Spark, where `VARIANCE` is correctly sample variance; T-SQL has no function named `VARIANCE` at all and needs `VAR(x)` instead |
| Trino / Presto / Hive | `PERCENTILE_CONT` / `approx_percentile` (dialect- and exactness-dependent) | native | `variance` is correctly sample variance per Trino/Presto docs |
This table is deliberately not exhaustive, Superset has roughly 75
`db_engine_specs` files. The proposed default (`BaseEngineSpec`) should be
the safe choice (mark unsupported, surface a clear user-facing error) rather
than an optimistic one, with individual engine specs opting in once
verified. Ship for the handful of engines above first, extend
opportunistically.
**`Count Unique Values`, `First`, `Last`, `List Unique Values`, explicitly
out of scope for this SIP:**
- `Count Unique Values` needs no work, it is already `COUNT_DISTINCT`.
- `First`/`Last` have no well-defined, unambiguous meaning as a plain
`GROUP BY` aggregate without an explicit ordering; most engines only
support this via window functions (`FIRST_VALUE`/`LAST_VALUE` `OVER
(ORDER BY ...)`) or do not support it as a simple aggregate at all
(Postgres has neither built in). Restoring this properly would mean
designing an "order by" sub-control on the metric, a real, separate
feature, not a one-line aggregate addition. Proposed as a follow-up SIP if
there is demand.
- `List Unique Values` maps to the `STRING_AGG`/`GROUP_CONCAT`/`LISTAGG`/
`ARRAY_AGG(DISTINCT ...)` family, real dialect differences, plus an open
UX question (unbounded cell content for high-cardinality columns).
Proposed as a follow-up SIP.
## New or changed public interfaces
- New `Aggregate` values (`MEDIAN`, `STDDEV_SAMP`, `VAR_SAMP`) selectable
anywhere the standard metric control appears, every chart type, not just
Pivot Table.
- New `BaseEngineSpec` extensibility point for per-engine aggregate SQL
generation (exact shape TBD in implementation, likely mirrors
`_time_grain_expressions`).
- No REST API surface changes beyond the existing metric aggregate field
accepting new values.
## Migration plan and compatibility
No new tables/columns needed for the aggregate addition itself.
Restoring prior chart settings, the way #42761 restored `show_values_as` for
charts that had it before #41184, is murkier here than it was for that PR
and needs its own design pass: the old `aggregate_function` was a single
Pivot-Table-level setting applied uniformly to every metric on the chart,
not a per-metric property. A chart that had `aggregate_function: Median`
before #41184, with a metric of `SUM(sales)`, was already silently wrong
under the old architecture (that is the bug that was fixed); mechanically
rewriting its metric to `MEDIAN(sales)` on upgrade would change what the
chart's leaf cells display, not just its totals, which may not match user
intent. This SIP proposes a best-effort, flagged-for-review migration
(surface affected charts to admins rather than silently rewriting them)
rather than a fully automatic one-to-one restoration.
## Rejected alternatives
- **Restoring the old `aggregateFunction` Pivot-Table control as-is.**
Rejected: this is the literal mechanism SIP-216 removed because it
reintroduces incorrect totals for non-additive metrics. Any fix has to go
through the metric's own aggregate, not a separate pivot-level override.
- **Routing all metric SQL generation through `sqlglot` expression-building
instead of SQLAlchemy's `sa.func`.** More architecturally thorough (would
give correct dialect syntax for free across more of the roughly 75 engine
specs), but a much larger, more invasive change to a hot path used by
every chart query. Noted as a possible future direction, not this SIP's
scope; this SIP proposes the smaller, `supports_grouping_sets`-shaped
extensibility point instead.
## Open questions
- **Resolved for this PR, worth confirming as the community's preferred
shape:** implemented as `BaseEngineSpec._extended_aggregations` (a
`{aggregate_name: sqla_column -> sqla_column}` dict) plus a
`get_extended_aggregation_func` accessor, set on the concrete or shared
base engine spec class per engine (e.g. on `PostgresBaseEngineSpec` so
Redshift inherits it, but *not* on `PrestoBaseEngineSpec` so Hive/Spark/
Databricks don't silently inherit unverified behavior, mirroring how
`supports_grouping_sets` is opted into per-concrete-engine there today).
Did not route through the `superset/sql/dialects/` sqlglot-based layer;
that layer is for SQL Lab parsing, wiring it into chart-metric query
building felt like a separate, larger change from this SIP's scope.
- **Still open, not addressed in this PR:** how aggressively should
`MEDIAN` degrade on engines without a native or exact equivalent?
Trino/Presto/Hive were left unimplemented (unsupported) specifically to
avoid silently answering this with an approximate function
(`approx_percentile`) that changes the semantics of what a user asked
for. If someone wants `MEDIAN` on those engines, this needs a real
decision: require explicit opt-in, show a UI warning, or keep it
disallowed.
- **Resolved for this PR:** left the two `sqla_aggregations` dicts
(`connectors/sqla/models.py` and `models/helpers.py`) unconsolidated,
both now separately wired to the same new `get_extended_aggregation_func`
hook. Consolidating them into one source of truth is real but unrelated
cleanup, not bundled here to keep the diff reviewable.
- **New, from implementation:** only Postgres, MySQL (partial), DuckDB, and
Redshift (by inheritance, unverified) ship enabled. BigQuery, Snowflake,
Trino, Presto, Hive, Spark, Databricks, Oracle, and T-SQL all have
documented (not live-verified) support per the table above but are not
yet wired up, each needs the same live-instance verification treatment
before being enabled, this PR intentionally didn't guess.
+1 -1
View File
@@ -14,7 +14,7 @@
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License
under the License.
-->
# Change Log
@@ -25,11 +25,8 @@ export type Aggregate =
| 'COUNT'
| 'COUNT_DISTINCT'
| 'MAX'
| 'MEDIAN'
| 'MIN'
| 'STDDEV_SAMP'
| 'SUM'
| 'VAR_SAMP';
| 'SUM';
export interface AdhocMetricBase {
hasCustomLabel?: boolean;
@@ -290,25 +290,6 @@ test('isAdditiveMetric: non-additive aggregates, SQL, and saved metrics are not
expect(isAdditiveMetric('count')).toBe(false);
});
test('isAdditiveMetric: MEDIAN/STDDEV_SAMP/VAR_SAMP are non-additive, with no dedicated code needed', () => {
// Regression guard: MEDIAN/STDDEV_SAMP/VAR_SAMP are new system-wide metric
// aggregates (not pivot-table-specific). They must fall outside
// ADDITIVE_AGGREGATES so totals/subtotals route through the correct
// DB-rollup path automatically, same as AVG/COUNT_DISTINCT already do --
// averaging per-group medians (or variances) is exactly the class of bug
// SIP-216 fixed for AVG, and would be equally wrong here.
(['MEDIAN', 'STDDEV_SAMP', 'VAR_SAMP'] as const).forEach(aggregate => {
expect(
isAdditiveMetric({
expressionType: 'SIMPLE',
aggregate,
column: { column_name: 'num' },
label: `${aggregate.toLowerCase()}_num`,
} as QueryFormMetric),
).toBe(false);
});
});
test('allMetricsAdditive: all additive vs any non-additive vs empty', () => {
const sum = {
expressionType: 'SIMPLE',
@@ -43,6 +43,23 @@ describe('SaveDatasetActionButton', () => {
expect(saveDatasetBtn).toBeVisible();
});
test('disables only the dataset button when canSaveDataset is false', () => {
const onSaveAsExplore = jest.fn();
render(
<SaveDatasetActionButton
setShowSave={() => true}
onSaveAsExplore={onSaveAsExplore}
canSaveDataset={false}
/>,
);
// Saving the query needs no results.
expect(screen.getByRole('button', { name: 'Save' })).toBeEnabled();
expect(
screen.getByRole('button', { name: /save dataset/i }),
).toBeDisabled();
});
test('disables the save dataset button when the query did not run successfully', async () => {
render(
<SaveDatasetActionButton
@@ -19,12 +19,14 @@
import { act, type ComponentProps } from 'react';
import {
cleanup,
createStore,
fireEvent,
render,
screen,
userEvent,
waitFor,
} from 'spec/helpers/testing-library';
import reducerIndex from 'spec/helpers/reducerIndex';
import fetchMock from 'fetch-mock';
import { SaveDatasetModal } from 'src/SqlLab/components/SaveDatasetModal';
import { createDatasource } from 'src/SqlLab/actions/sqlLab';
@@ -63,6 +65,12 @@ beforeEach(() => {
cleanup();
});
afterEach(() => {
// In-body restores are skipped when an assertion throws, leaking a
// configured spy into later tests.
jest.restoreAllMocks();
});
// Mock createDatasource to return a thunk that resolves with the dataset's
// new id. The test's mock store includes redux-thunk middleware (from RTK's
// getDefaultMiddleware), so dispatch(createDatasource(...)) properly unwraps
@@ -518,6 +526,39 @@ describe('SaveDatasetModal', () => {
});
});
test('surfaces the error and keeps the modal open when saving fails', async () => {
// The chart-payload step's toast was built but never dispatched, so a
// failure there was silent.
const postFormData = jest.spyOn(
require('src/explore/exploreUtils/formData'),
'postFormData',
);
postFormData.mockRejectedValue(new Error('Boom'));
const onHide = jest.fn();
const store = createStore({ user }, reducerIndex);
render(<SaveDatasetModal {...mockedProps} onHide={onHide} />, { store });
fireEvent.change(screen.getByDisplayValue(/unimportant/i), {
target: { value: 'my dataset' },
});
userEvent.click(screen.getByRole('button', { name: /save/i }));
// `createStore` builds its reducer map at runtime, so state isn't typed.
const toasts = () =>
(
store.getState() as unknown as {
messageToasts: { toastType: string }[];
}
).messageToasts;
await waitFor(() => {
expect(toasts()).toHaveLength(1);
});
expect(toasts()[0].toastType).toBe('DANGER_TOAST');
expect(onHide).not.toHaveBeenCalled();
});
test('clearDatasetCache is imported and available', () => {
const { clearDatasetCache } = require('src/utils/cachedSupersetGet');
@@ -61,6 +61,9 @@ import type Subject from 'src/types/Subject';
import { openInNewTab, redirect } from 'src/utils/navigationUtils';
import { mapSubjectValuesToIds } from 'src/features/subjects/SubjectPicker';
// Derived so it can't drift from what `getClientErrorObject` accepts.
type SaveErrorSource = Parameters<typeof getClientErrorObject>[0];
interface QueryDatabase {
id?: number;
}
@@ -391,9 +394,18 @@ export const SaveDatasetModal = ({
setDatasetName(getDefaultDatasetName());
onHide();
})
.catch(() => {
.catch((error?: SaveErrorSource) => {
setLoading(false);
addDangerToast(t('An error occurred saving dataset'));
// `createDatasource` already toasted the server's message and rejects
// with nothing; only the chart-payload step needs its own.
if (!error) {
return;
}
getClientErrorObject(error).then(e =>
dispatch(
addDangerToast(e.error || t('An error occurred saving dataset')),
),
);
});
};
@@ -27,6 +27,8 @@ import {
import SaveQuery from 'src/SqlLab/components/SaveQuery';
import { initialState, databases } from 'src/SqlLab/fixtures';
const RESULT_COLUMNS = [{ column_name: 'col', type: 'STRING' }];
const mockedProps = {
queryEditorId: '123',
animation: false,
@@ -35,7 +37,6 @@ const mockedProps = {
onSave: () => {},
saveQueryWarning: null,
columns: [],
canSaveDataset: true,
};
const mockState = {
@@ -60,8 +61,31 @@ const splitSaveBtnProps = {
...mockedProps.database,
allows_virtual_table_explore: true,
},
columns: RESULT_COLUMNS,
};
const EDITOR_SQL = 'SELECT * FROM t';
const stateWithLatestQuery = ({
id,
state,
sql = EDITOR_SQL,
}: {
id: string;
state: string;
sql?: string;
}) => ({
...mockState,
sqlLab: {
...mockState.sqlLab,
queryEditors: mockState.sqlLab.queryEditors.map(qe => ({
...qe,
latestQueryId: id,
})),
queries: { [id]: { id, state, sql } },
},
});
const middlewares = [thunk];
const mockStore = configureStore(middlewares);
@@ -97,6 +121,71 @@ describe('SavedQuery', () => {
expect(saveBtn).toBeVisible();
});
test('blocks "Save dataset" until the query has run successfully', () => {
// Without a successful run the save can only fail server-side.
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'failed' })),
});
expect(
screen.getByRole('button', { name: /save dataset/i }),
).toBeDisabled();
// Saving the query itself is unaffected.
expect(screen.getByRole('button', { name: 'Save' })).toBeEnabled();
});
test('blocks "Save dataset" when no query has been run at all', () => {
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(mockState),
});
expect(
screen.getByRole('button', { name: /save dataset/i }),
).toBeDisabled();
});
test('blocks "Save dataset" when the SQL changed after a successful run', () => {
// The run succeeded, but not for what is in the editor now -- and it is
// the editor's SQL that gets saved.
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(
stateWithLatestQuery({
id: 'qid-1',
state: 'success',
sql: 'SELECT 1 AS ran_earlier',
}),
),
});
expect(
screen.getByRole('button', { name: /save dataset/i }),
).toBeDisabled();
});
test('blocks "Save dataset" when the successful query returned no columns', () => {
// e.g. a DDL/DML statement -- there is nothing to introspect into a dataset.
render(<SaveQuery {...splitSaveBtnProps} columns={[]} />, {
useRedux: true,
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
});
expect(
screen.getByRole('button', { name: /save dataset/i }),
).toBeDisabled();
});
test('enables "Save dataset" once the query has succeeded', () => {
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
});
expect(screen.getByRole('button', { name: /save dataset/i })).toBeEnabled();
});
test('renders a save query modal when user clicks save button', () => {
render(<SaveQuery {...mockedProps} />, {
useRedux: true,
@@ -234,7 +323,7 @@ describe('SavedQuery', () => {
test('renders a save dataset modal when user clicks "save dataset" menu item', async () => {
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(mockState),
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
});
const saveDatasetMenuItem = await screen.findByLabelText(/save dataset/i);
@@ -248,7 +337,7 @@ describe('SavedQuery', () => {
test('renders the save dataset modal UI', async () => {
render(<SaveQuery {...splitSaveBtnProps} />, {
useRedux: true,
store: mockStore(mockState),
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
});
const saveDatasetMenuItem = await screen.findByLabelText(/save dataset/i);
userEvent.click(saveDatasetMenuItem);
@@ -17,6 +17,8 @@
* under the License.
*/
import { useState, useEffect, useMemo, ChangeEvent } from 'react';
import { useSelector } from 'react-redux';
import { Query, QueryState } from '@superset-ui/core';
import type { DatabaseObject } from 'src/features/databases/types';
import { t } from '@apache-superset/core/translation';
import { styled } from '@apache-superset/core/theme';
@@ -37,7 +39,7 @@ import {
} from 'src/SqlLab/components/SaveDatasetModal';
import { getDatasourceAsSaveableDataset } from 'src/utils/datasourceUtils';
import useQueryEditor from 'src/SqlLab/hooks/useQueryEditor';
import { QueryEditor } from 'src/SqlLab/types';
import { QueryEditor, SqlLabRootState } from 'src/SqlLab/types';
import useLogAction from 'src/logger/useLogAction';
import {
LOG_ACTIONS_SQLLAB_CREATE_CHART,
@@ -52,7 +54,6 @@ interface SaveQueryProps {
onUpdate: (arg0: QueryPayload, id: string) => void;
saveQueryWarning: string | null;
database: Partial<DatabaseObject> | undefined;
canSaveDataset: boolean;
}
export type QueryPayload = {
@@ -82,7 +83,6 @@ const SaveQuery = ({
saveQueryWarning,
database,
columns,
canSaveDataset,
}: SaveQueryProps) => {
const queryEditor = useQueryEditor(queryEditorId, [
'autorun',
@@ -113,6 +113,17 @@ const SaveQuery = ({
const [label, setLabel] = useState<string>(defaultLabel);
const [showSave, setShowSave] = useState<boolean>(false);
const [showSaveDatasetModal, setShowSaveDatasetModal] = useState(false);
// Saving a dataset runs the SQL to introspect columns, so it needs a
// successful run of the SQL being saved that produced at least one column
// -- editing after a run invalidates it, and running a selection only
// validates that selection.
const latestQuery = useSelector<SqlLabRootState, Query | undefined>(
({ sqlLab }) => sqlLab.queries[queryEditor.latestQueryId || ''],
);
const canSaveDataset =
latestQuery?.state === QueryState.Success &&
latestQuery.sql === queryEditor.sql &&
columns.length > 0;
const isSaved = !!query.remoteId;
const isLabelEmpty = label.trim().length === 0;
const canExploreDatabase = !!database?.allows_virtual_table_explore;
@@ -356,7 +356,10 @@ describe('SqlEditor', () => {
);
test('enables the save dataset button when the latest query succeeded', async () => {
const { findByRole } = setupWithLatestQuery({ state: QueryState.Success });
const { findByRole } = setupWithLatestQuery({
state: QueryState.Success,
sql: mockedProps.queryEditor.sql,
});
expect(await findByRole('button', { name: 'Save dataset' })).toBeEnabled();
});
@@ -868,7 +868,6 @@ const SqlEditor: FC<Props> = ({
}
saveQueryWarning={saveQueryWarning}
database={database}
canSaveDataset={successful && resultColumns.length > 0}
/>
<ShareSqlLabQuery queryEditorId={queryEditor.id} />
</>
@@ -191,29 +191,6 @@ describe('AdhocMetric', () => {
expect(adhocMetric2.inferSqlExpressionAggregate()).toBeNull();
});
test('can infer the new extended aggregates (STDDEV_SAMP/VAR_SAMP/MEDIAN) from sql expressions', () => {
const stddevSamp = new AdhocMetric({
expressionType: EXPRESSION_TYPES.SQL,
sqlExpression: 'STDDEV_SAMP(my_column)',
});
expect(stddevSamp.inferSqlExpressionColumn()).toBe('my_column');
expect(stddevSamp.inferSqlExpressionAggregate()).toBe('STDDEV_SAMP');
const varSamp = new AdhocMetric({
expressionType: EXPRESSION_TYPES.SQL,
sqlExpression: 'VAR_SAMP(my_column)',
});
expect(varSamp.inferSqlExpressionColumn()).toBe('my_column');
expect(varSamp.inferSqlExpressionAggregate()).toBe('VAR_SAMP');
const median = new AdhocMetric({
expressionType: EXPRESSION_TYPES.SQL,
sqlExpression: 'MEDIAN(my_column)',
});
expect(median.inferSqlExpressionColumn()).toBe('my_column');
expect(median.inferSqlExpressionAggregate()).toBe('MEDIAN');
});
test('will infer columns and aggregates when converting to a simple expression', () => {
const adhocMetric = new AdhocMetric({
expressionType: EXPRESSION_TYPES.SQL,
@@ -271,20 +248,4 @@ describe('AdhocMetric', () => {
).toBe('COUNT_DISTINCT');
expect(emptyColumnName.getDefaultLabel()).toBe('COUNT_DISTINCT');
});
test('should prefill a portable MEDIAN expression for the Custom SQL tab, but keep the raw label', () => {
const median = new AdhocMetric({
column: valueColumn,
aggregate: AGGREGATES.MEDIAN,
hasCustomLabel: false,
});
// MEDIAN(column) isn't valid SQL on every engine this PR verifies it
// for (e.g. PostgreSQL has no MEDIAN function), so the editable Custom
// SQL tab is prefilled with the portable, standards-based spelling.
expect(median.translateToSql({ transformCountDistinct: true })).toBe(
'PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value)',
);
// The display label stays the concise, human-readable form.
expect(median.getDefaultLabel()).toBe('MEDIAN(value)');
});
});
@@ -153,19 +153,6 @@ export default class AdhocMetric {
) {
return `COUNT(DISTINCT ${column.slice(1, -1)})`;
}
// MEDIAN(column) isn't a real function on every engine this PR
// verifies it for -- PostgreSQL/Redshift compile it to
// PERCENTILE_CONT(0.5) WITHIN GROUP instead. `transformCountDistinct`
// signals this call is prefilling the *editable, executable* Custom
// SQL tab (not just a display label), so use the portable,
// standards-based spelling there instead of the raw aggregate name.
if (
params.transformCountDistinct &&
aggregate === AGGREGATES.MEDIAN &&
/^\(.*\)$/.test(column)
) {
return `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${column.slice(1, -1)})`;
}
return aggregate + column;
}
if (this.expressionType === EXPRESSION_TYPES.SQL) {
@@ -36,7 +36,6 @@ import {
import sqlKeywords from 'src/SqlLab/utils/sqlKeywords';
import { noOp } from 'src/utils/common';
import {
AGGREGATES_LABELS,
AGGREGATES_OPTIONS,
POPOVER_INITIAL_HEIGHT,
POPOVER_INITIAL_WIDTH,
@@ -549,7 +548,7 @@ function AdhocMetricEditPopover({
<Select
options={AGGREGATES_OPTIONS.map(option => ({
value: option,
label: AGGREGATES_LABELS[option] ?? option,
label: option,
key: option,
}))}
{...aggregateSelectProps}
+1 -12
View File
@@ -23,22 +23,11 @@ export const AGGREGATES = {
COUNT: 'COUNT',
COUNT_DISTINCT: 'COUNT_DISTINCT',
MAX: 'MAX',
MEDIAN: 'MEDIAN',
MIN: 'MIN',
STDDEV_SAMP: 'STDDEV_SAMP',
SUM: 'SUM',
VAR_SAMP: 'VAR_SAMP',
};
export const AGGREGATES_OPTIONS = Object.values(AGGREGATES);
// Human-readable labels for aggregates whose raw enum value isn't
// self-explanatory in the UI. Aggregates absent here (AVG, COUNT, MAX,
// MEDIAN, MIN, SUM, ...) are already clear as their raw value.
export const AGGREGATES_LABELS: Record<string, string> = {
STDDEV_SAMP: t('Sample Standard Deviation'),
VAR_SAMP: t('Sample Variance'),
};
export enum Operators {
Equals = 'EQUALS',
NotEquals = 'NOT_EQUALS',
@@ -187,7 +176,7 @@ export const DISABLE_INPUT_OPERATORS = [
export const sqlaAutoGeneratedMetricNameRegex =
/^(sum|min|max|avg|count|count_distinct)__.*$/i;
export const sqlaAutoGeneratedMetricRegex =
/^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT|MEDIAN|STDDEV_SAMP|VAR_SAMP)\([A-Z0-9_."]*\)$/i;
/^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
export const TIME_FILTER_LABELS = {
time_range: t('Time range'),
-6
View File
@@ -413,12 +413,6 @@ class ChartDataRestApi(ChartRestApi):
# for async queries with jinja context
set_form_data(cached_data)
query_context = self._create_query_context_from_form(cached_data)
# Mark as a cache replay so _sql_filters_modified skips the
# SQL-extras check. The original request already passed the
# full security check, cache keys are opaque SHA-256 hashes
# (unguessable), and force_cached only serves pre-computed
# data — no new SQL is executed.
query_context._from_cache_replay = True
command = ChartDataCommand(query_context)
command.validate()
except ChartDataCacheLoadError:
+1 -10
View File
@@ -42,7 +42,6 @@ from superset.utils import pandas_postprocessing, schema as utils
from superset.utils.core import (
AnnotationType,
DatasourceType,
EXTENDED_METRIC_AGGREGATES,
FilterOperator,
PostProcessingBoxplotWhiskerType,
PostProcessingContributionOrientation,
@@ -435,15 +434,7 @@ class ChartDataAdhocMetricSchema(Schema):
"Only required for simple expression types."
},
validate=validate.OneOf(
choices=(
"AVG",
"COUNT",
"COUNT_DISTINCT",
"MAX",
"MIN",
"SUM",
*sorted(EXTENDED_METRIC_AGGREGATES),
)
choices=("AVG", "COUNT", "COUNT_DISTINCT", "MAX", "MIN", "SUM")
),
)
column = fields.Nested(ChartDataColumnSchema)
+24 -2
View File
@@ -33,7 +33,11 @@ from superset.commands.dataset.exceptions import (
)
from superset.commands.utils import populate_subjects
from superset.daos.dataset import DatasetDAO
from superset.exceptions import SupersetParseError, SupersetSecurityException
from superset.exceptions import (
SupersetException,
SupersetParseError,
SupersetSecurityException,
)
from superset.extensions import security_manager
from superset.sql.parse import Table
from superset.utils.decorators import on_error, transaction
@@ -50,7 +54,25 @@ class CreateDatasetCommand(CreateMixin, BaseCommand):
self.validate()
dataset = DatasetDAO.create(attributes=self._properties)
dataset.fetch_metadata()
try:
dataset.fetch_metadata()
except SupersetException as ex:
# Not a SQLAlchemyError, so ``on_error`` re-raises it untouched and
# it escapes to FAB's ``@safe`` as an opaque 500 "Fatal error".
# Deliberately covers the 403 ``SupersetSecurityException`` raised
# for mutation/multi-statement SQL too: ``validate()`` already
# reports that class of rejection as a 422 on ``sql`` via
# ``DatasetDataAccessIsNotAllowed``.
raise DatasetInvalidError(
exceptions=[
ValidationError(
# ``lazy_gettext`` messages aren't ``str``, so
# marshmallow won't wrap them into a list on its own.
[str(ex.message)],
field_name="sql" if self._properties.get("sql") else "table",
)
]
) from ex
return dataset
def validate(self) -> None: # noqa: C901
+6 -21
View File
@@ -1898,6 +1898,11 @@ class SqlaTable(
if expression_type == utils.AdhocMetricExpressionType.SIMPLE:
aggregate: Any = metric.get("aggregate")
if (
not isinstance(aggregate, str)
or aggregate not in self.sqla_aggregations
):
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
metric_column = metric.get("column") or {}
column_name = cast(str, metric_column.get("column_name"))
table_column: TableColumn | None = columns_by_name.get(column_name)
@@ -1907,27 +1912,7 @@ class SqlaTable(
)
else:
sqla_column = column(column_name)
if isinstance(aggregate, str) and aggregate in self.sqla_aggregations:
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
elif isinstance(aggregate, str) and (
extended_func := self.db_engine_spec.get_extended_aggregation_func(
aggregate
)
):
sqla_metric = extended_func(sqla_column)
elif (
isinstance(aggregate, str)
and aggregate in utils.EXTENDED_METRIC_AGGREGATES
):
raise QueryObjectValidationError(
_(
"The %(aggregate)s aggregate is not supported on this database",
aggregate=aggregate,
)
)
else:
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
elif expression_type == utils.AdhocMetricExpressionType.SQL:
expression: str | None = metric.get("sqlExpression")
if not isinstance(expression, str) or not expression.strip():
-27
View File
@@ -632,33 +632,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# issuing one query per level. Conservative default of False; engines opt in.
supports_grouping_sets = False
# SQL-generating callables for metric aggregates that have no safe, universal
# cross-dialect spelling -- unlike SUM/COUNT/AVG/MIN/MAX/COUNT_DISTINCT (see
# `SqlaTable.sqla_aggregations`), which SQLAlchemy's generic `sa.func` can emit
# unchanged on every engine. Keyed by `Aggregate` name (see
# `superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`);
# each value takes a SQLAlchemy column and returns the aggregate expression.
# Absent by default: an aggregate not present here is unsupported on this
# engine, and callers must surface a clear "not supported" error rather than
# emit unverified SQL (a wrong statistic returned silently is worse than an
# error). Engines opt in via `get_extended_aggregation_func` below once the
# expression has been verified against real engine behavior, not assumed
# from syntax alone -- see the MySQL engine spec for a concrete example of
# why this distinction matters (its `VARIANCE()` computes the *population*
# variance, not the *sample* variance `VAR_SAMP` denotes).
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
@classmethod
def get_extended_aggregation_func(
cls, aggregate: str
) -> Callable[[ColumnElement], ColumnElement] | None:
"""
SQL-generating callable for an aggregate not handled by the generic
`sa.func` mapping (e.g. MEDIAN, STDDEV_SAMP, VAR_SAMP). Returns None if
this engine has no verified, correct expression for it.
"""
return cls._extended_aggregations.get(aggregate)
# Is the DB engine spec able to change the default schema? This requires implementing # noqa: E501
# a custom `adjust_engine_params` method.
supports_dynamic_schema = False
+1 -8
View File
@@ -15,10 +15,9 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Any, Callable, Optional
from typing import Any, Optional
from sqlalchemy import types
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresEngineSpec
@@ -28,12 +27,6 @@ class CockroachDbEngineSpec(PostgresEngineSpec):
engine = "cockroachdb"
engine_name = "CockroachDB"
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not CockroachDB's distributed
# query engine; disable it here until someone confirms the same expressions
# against a live CockroachDB instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"CockroachDB is a distributed SQL database built for cloud applications."
+1 -7
View File
@@ -17,14 +17,13 @@
import logging
import re
from re import Pattern
from typing import Any, Callable, Optional
from typing import Any, Optional
from urllib import parse
from flask_babel import gettext as __
from sqlalchemy import Float, Integer, Numeric, String, TEXT, text, types
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.type_api import TypeEngine
from superset.db_engine_specs.base import DatabaseCategory
@@ -122,11 +121,6 @@ class DorisEngineSpec(MySQLEngineSpec):
# while technically supported by Doris, this generates invalid table identifiers
supports_cross_catalog_queries = False
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
# against real MySQL behavior, not Doris's OLAP query engine; disable it here
# until someone confirms the same expressions against a live Doris instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"Apache Doris is a high-performance real-time analytical database."
+1 -15
View File
@@ -20,9 +20,8 @@ from __future__ import annotations
import re
from datetime import datetime
from re import Pattern
from typing import Any, Callable, TYPE_CHECKING, TypedDict
from typing import Any, TYPE_CHECKING, TypedDict
import sqlalchemy as sa
from apispec import APISpec
from apispec.ext.marshmallow import MarshmallowPlugin
from flask import current_app as app
@@ -31,7 +30,6 @@ from marshmallow import fields, Schema
from sqlalchemy import text, types
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from superset.constants import TimeGrain
from superset.databases.utils import make_url_safe
@@ -204,18 +202,6 @@ class DuckDBEngineSpec(DuckDBParametersMixin, BaseEngineSpec):
sqlalchemy_uri_placeholder = "duckdb:////path/to/duck.db"
supports_multivalues_insert = True
# Verified against a live duckdb instance (in-process, no server needed),
# including under GROUPING SETS: the grand total correctly reflects every
# row, not an aggregate-of-aggregates. STDDEV_SAMP/VAR_SAMP values match
# postgres/mysql exactly for the same inputs; MEDIAN matches postgres
# (mysql has no native MEDIAN to compare against). Inherited by
# MotherDuckEngineSpec.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
"MEDIAN": sa.func.median,
"STDDEV_SAMP": sa.func.stddev_samp,
"VAR_SAMP": sa.func.var_samp,
}
metadata = {
"description": (
"DuckDB is an in-process OLAP database designed for fast "
-10
View File
@@ -15,10 +15,6 @@
# specific language governing permissions and limitations
# under the License.
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresEngineSpec
@@ -34,12 +30,6 @@ class GreenplumEngineSpec(PostgresEngineSpec):
engine_name = "Greenplum"
default_driver = "psycopg2"
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is
# verified against real Postgres behavior, not Greenplum's MPP query engine;
# disable it here until someone confirms the same expressions against a live
# Greenplum instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"VMware Greenplum is a massively parallel processing (MPP) "
+1 -7
View File
@@ -15,10 +15,9 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Any, Callable, Optional
from typing import Any, Optional
from sqlalchemy import types
from sqlalchemy.sql.elements import ColumnElement
from superset.constants import TimeGrain
from superset.db_engine_specs.base import DatabaseCategory
@@ -30,11 +29,6 @@ class HanaEngineSpec(PostgresBaseEngineSpec):
engine = "hana"
engine_name = "SAP HANA"
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not HANA's; disable it here
# until someone confirms the same expressions against a live HANA instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"SAP HANA is an in-memory relational database and application platform."
-10
View File
@@ -17,10 +17,6 @@
from __future__ import annotations
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
@@ -36,12 +32,6 @@ class HologresEngineSpec(PostgresBaseEngineSpec):
engine_name = "Hologres"
default_driver = "psycopg2"
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not Hologres's real-time analytics
# engine; disable it here until someone confirms the same expressions against a
# live Hologres instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"Alibaba Cloud Hologres is a real-time interactive analytics service, "
-18
View File
@@ -25,7 +25,6 @@ from re import Pattern
from typing import Any, Callable, Optional, TYPE_CHECKING
from urllib import parse
import sqlalchemy as sa
from flask_babel import gettext as __
from sqlalchemy import types
from sqlalchemy.dialects.mysql import (
@@ -41,7 +40,6 @@ from sqlalchemy.dialects.mysql import (
TINYTEXT,
)
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from superset.constants import TimeGrain
from superset.db_engine_specs.base import (
@@ -96,22 +94,6 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
supports_dynamic_schema = True
supports_multivalues_insert = True
# Verified against a live mysql:8.0 instance, including under GROUP BY ...
# WITH ROLLUP. `STDDEV_SAMP`/`VAR_SAMP` are native, correct sample
# statistics. MEDIAN is deliberately absent: MySQL has neither a `MEDIAN`
# function nor `PERCENTILE_CONT` (confirmed: both error). Its `VARIANCE()`
# function is population variance, not sample variance, so it is not a
# valid stand-in for VAR_SAMP either.
# Inherited by MariaDB (a MySQL fork implementing the same aggregate
# functions) and by Aurora MySQL / its Data API variant (AWS's wire- and
# SQL-compatible managed MySQL) -- unlike CockroachDB/Greenplum/HANA
# relative to Postgres, none of these run a materially different query
# engine, so no separate reset is needed.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
"STDDEV_SAMP": sa.func.stddev_samp,
"VAR_SAMP": sa.func.var_samp,
}
metadata = {
"description": "MySQL is a popular open-source relational database.",
"logo": "mysql.png",
-9
View File
@@ -14,10 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.constants import TimeGrain
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
@@ -28,11 +24,6 @@ class NetezzaEngineSpec(PostgresBaseEngineSpec):
default_driver = "nzpy"
engine_name = "IBM Netezza Performance Server"
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not Netezza's; disable it here
# until someone confirms the same expressions against a live Netezza instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": "IBM Netezza Performance Server is a data warehouse appliance.",
"logo": "netezza.png",
+1 -8
View File
@@ -17,11 +17,10 @@
import logging
import re
from re import Pattern
from typing import Any, Callable, Optional
from typing import Any, Optional
from flask_babel import gettext as __
from sqlalchemy import Numeric, TEXT, types
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.type_api import TypeEngine
from superset.db_engine_specs.base import DatabaseCategory
@@ -85,12 +84,6 @@ class OceanBaseEngineSpec(MySQLEngineSpec):
encryption_parameters = {"ssl": "0"}
supports_dynamic_schema = True
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
# against real MySQL behavior, not OceanBase's distributed query engine;
# disable it here until someone confirms the same expressions against a live
# OceanBase instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": "OceanBase is a distributed relational database.",
"logo": "oceanbase.svg",
-21
View File
@@ -23,14 +23,12 @@ from datetime import datetime
from re import Pattern
from typing import Any, Callable, Optional, TYPE_CHECKING
import sqlalchemy as sa
from flask_babel import gettext as __
from sqlalchemy import text, types
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, INTERVAL, JSON
from sqlalchemy.dialects.postgresql.base import PGInspector
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.expression import ColumnClause
from sqlalchemy.types import Date, DateTime, String
@@ -194,25 +192,6 @@ class PostgresBaseEngineSpec(BaseEngineSpec):
TimeGrain.YEAR: "DATE_TRUNC('year', {col})",
}
# Verified against a live postgres:16 instance, including under GROUPING
# SETS (the pivot table's non-additive-total rollup pattern): the grand
# total correctly reflects every row, not an aggregate-of-aggregates.
# STDDEV_SAMP/VAR_SAMP (not MEDIAN -- see its override) are inherited by
# Redshift (a Postgres fork); its SQL function reference documents the
# same support, but that has not been separately verified against a live
# Redshift instance.
# Also inherited by TimescaleDB (a Postgres extension, not a forked query
# engine -- it runs unmodified Postgres aggregate execution) and by
# Aurora PostgreSQL / its Data API variant (AWS's wire- and
# SQL-compatible managed Postgres). Engines that share the SQL dialect
# but run a materially different query engine (CockroachDB, Greenplum,
# SAP HANA) reset this to `{}` instead -- see those engine specs.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
"MEDIAN": lambda col: sa.func.percentile_cont(0.5).within_group(col),
"STDDEV_SAMP": sa.func.stddev_samp,
"VAR_SAMP": sa.func.var_samp,
}
custom_errors: dict[Pattern[str], tuple[str, SupersetErrorType, dict[str, Any]]] = {
CONNECTION_INVALID_USERNAME_REGEX: (
__('The username "%(username)s" does not exist.'),
+1 -21
View File
@@ -20,12 +20,10 @@ import logging
import re
import warnings
from re import Pattern
from typing import Any, Callable
from typing import Any
import pandas as pd
import sqlalchemy as sa
from flask_babel import gettext as __
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.types import NVARCHAR
from superset.db_engine_specs.base import BasicParametersMixin, DatabaseCategory
@@ -279,24 +277,6 @@ class RedshiftEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec):
"$.aws_iam.role_arn": "AWS IAM Role ARN",
}
# Redshift inherits `PostgresBaseEngineSpec._extended_aggregations` for
# STDDEV_SAMP/VAR_SAMP (plain, non-sort-based aggregate calls), but
# overrides MEDIAN here instead of inheriting Postgres's spelling.
# Postgres has no native MEDIAN function and compiles it to
# `percentile_cont(0.5) WITHIN GROUP (ORDER BY col)`; Redshift, unlike
# Postgres, documents a native `MEDIAN(x)` aggregate function. Using
# that native spelling -- rather than the inherited WITHIN-GROUP form --
# also sidesteps a documented Redshift restriction rejecting more than
# one sort-based aggregate (MEDIAN, PERCENTILE_CONT, LISTAGG WITHIN
# GROUP, ...) with a different ORDER BY in the same query, e.g.
# `MEDIAN(sales)` alongside `MEDIAN(margin)`: a plain function call has
# no explicit ORDER BY clause to conflict.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
"MEDIAN": sa.func.median,
"STDDEV_SAMP": PostgresBaseEngineSpec._extended_aggregations["STDDEV_SAMP"],
"VAR_SAMP": PostgresBaseEngineSpec._extended_aggregations["VAR_SAMP"],
}
@staticmethod
def update_params_from_encrypted_extra(
database: Database,
-10
View File
@@ -14,10 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresEngineSpec
@@ -30,12 +26,6 @@ class RisingWaveDbEngineSpec(PostgresEngineSpec):
"risingwave://user:password@host:port/dbname[?key=value&key=value...]"
)
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is
# verified against real Postgres behavior, not RisingWave's streaming query
# engine; disable it here until someone confirms the same expressions against
# a live RisingWave instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": "RisingWave is a distributed streaming database.",
"logo": "risingwave.svg",
+1 -7
View File
@@ -20,7 +20,7 @@ import logging
import re
from datetime import datetime
from re import Pattern
from typing import Any, Callable, Optional, TYPE_CHECKING, TypedDict
from typing import Any, Optional, TYPE_CHECKING, TypedDict
from urllib import parse
from apispec import APISpec
@@ -33,7 +33,6 @@ from marshmallow import fields, Schema
from sqlalchemy import text, types
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from superset.constants import TimeGrain
from superset.databases.utils import make_url_safe
@@ -88,11 +87,6 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
force_column_alias_quotes = True
max_column_name_length = 256
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not Snowflake's; disable it here
# until someone confirms the same expressions against a live Snowflake instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
# Snowflake doesn't support IS true/false syntax, use = true/false instead
use_equality_for_boolean_filters = True
+1 -8
View File
@@ -18,14 +18,13 @@
import logging
import re
from re import Pattern
from typing import Any, Callable
from typing import Any
from urllib import parse
from flask_babel import gettext as __
from sqlalchemy import Float, Integer, Numeric, text, types
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.type_api import TypeEngine
from superset import is_feature_enabled
@@ -104,12 +103,6 @@ class StarRocksEngineSpec(MySQLEngineSpec):
supports_dynamic_schema = True
supports_catalog = supports_dynamic_catalog = supports_cross_catalog_queries = True
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
# against real MySQL behavior, not StarRocks's OLAP query engine; disable it
# here until someone confirms the same expressions against a live StarRocks
# instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"StarRocks is a high-performance analytical database "
-9
View File
@@ -14,10 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
@@ -26,11 +22,6 @@ class VerticaEngineSpec(PostgresBaseEngineSpec):
engine = "vertica"
engine_name = "Vertica"
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not Vertica's; disable it here
# until someone confirms the same expressions against a live Vertica instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": "Vertica is a column-oriented analytics database.",
"logo": "vertica.png",
-10
View File
@@ -17,10 +17,6 @@
from __future__ import annotations
from typing import Callable
from sqlalchemy.sql.elements import ColumnElement
from superset.db_engine_specs.base import DatabaseCategory
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
@@ -36,12 +32,6 @@ class YugabyteDBEngineSpec(PostgresBaseEngineSpec):
engine_name = "YugabyteDB"
default_driver = "psycopg2"
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
# is verified against real Postgres behavior, not YugabyteDB's distributed
# query engine; disable it here until someone confirms the same expressions
# against a live YugabyteDB instance.
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
metadata = {
"description": (
"YugabyteDB is a distributed SQL database built on top of PostgreSQL."
+3 -10
View File
@@ -607,19 +607,12 @@ def create_metric_object(col: ColumnRef) -> Dict[str, Any] | str:
"MIN",
"MAX",
"COUNT_DISTINCT",
"STDDEV_SAMP",
"VAR_SAMP",
"STDDEV",
"VAR",
"MEDIAN",
"PERCENTILE",
}
# Accept the pre-SIP shorthand names too, mapped onto the real,
# unambiguous aggregate names Superset actually supports (bare
# "STDDEV"/"VAR" are ambiguous between sample and population statistics,
# and differ by engine -- see docs/sip/median-stddev-variance-aggregates.md).
aggregate_aliases = {"STDDEV": "STDDEV_SAMP", "VAR": "VAR_SAMP"}
aggregate = aggregate_aliases.get(
(col.aggregate or "SUM").upper(), col.aggregate or "SUM"
)
aggregate = col.aggregate or "SUM"
# Validate aggregate function (final safety check)
if aggregate.upper() not in valid_aggregates:
@@ -137,9 +137,7 @@ Example table config:
- If the chart type doesn't suit the data, try a different kind
## Available Aggregations
SUM, COUNT, AVG, MIN, MAX, COUNT_DISTINCT, STDDEV_SAMP, VAR_SAMP, MEDIAN
(support for STDDEV_SAMP/VAR_SAMP/MEDIAN depends on the connected database;
an unsupported choice returns a clear error naming the unsupported aggregate)
SUM, COUNT, AVG, MIN, MAX, COUNT_DISTINCT, STDDEV, VAR, MEDIAN
## Custom SQL Metrics
For ratio metrics, weighted averages, and conditional aggregates,
+2 -7
View File
@@ -718,15 +718,10 @@ class ColumnRef(UnknownFieldCheckMixin):
"MIN",
"MAX",
"COUNT_DISTINCT",
"STDDEV_SAMP",
"VAR_SAMP",
"MEDIAN",
"PERCENTILE",
# Pre-SIP shorthand, accepted and normalized to the names above by
# `chart_utils.create_metric_object`; kept here so schema
# validation doesn't reject them before that normalization runs.
"STDDEV",
"VAR",
"MEDIAN",
"PERCENTILE",
]
| None
) = Field(None, description="SQL aggregate function")
@@ -693,15 +693,7 @@ class DatasetValidator:
# and text in most SQL engines, so restricting them here would
# produce false-positive errors. Leave those to the Tier-2
# compile check.
numeric_aggs = [
"SUM",
"AVG",
"STDDEV_SAMP",
"VAR_SAMP",
"MEDIAN",
"STDDEV",
"VAR",
]
numeric_aggs = ["SUM", "AVG", "STDDEV", "VAR", "MEDIAN"]
if (
col_ref.aggregate in numeric_aggs
and not col_info.get("is_numeric", False)
@@ -217,14 +217,7 @@ class FormatTypeValidator:
"""Suggest appropriate format based on column and aggregation."""
if column.aggregate in ["COUNT", "COUNT_DISTINCT"]:
return ",d" # Integer with thousands separator
elif column.aggregate in [
"AVG",
"MEDIAN",
"STDDEV_SAMP",
"VAR_SAMP",
"STDDEV",
"VAR",
]:
elif column.aggregate in ["AVG", "STDDEV", "VAR"]:
return ",.2f" # Two decimals for statistical measures
elif column.aggregate in ["SUM", "MIN", "MAX"]:
# Could be currency or regular number, default to flexible
+6 -21
View File
@@ -3518,30 +3518,15 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
if expression_type == utils.AdhocMetricExpressionType.SIMPLE:
aggregate: Any = metric.get("aggregate")
if (
not isinstance(aggregate, str)
or aggregate not in self.sqla_aggregations
):
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
metric_column = metric.get("column") or {}
column_name = cast(str, metric_column.get("column_name"))
sqla_column = sa.column(column_name)
if isinstance(aggregate, str) and aggregate in self.sqla_aggregations:
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
elif isinstance(aggregate, str) and (
extended_func := self.db_engine_spec.get_extended_aggregation_func(
aggregate
)
):
sqla_metric = extended_func(sqla_column)
elif (
isinstance(aggregate, str)
and aggregate in utils.EXTENDED_METRIC_AGGREGATES
):
raise QueryObjectValidationError(
_(
"The %(aggregate)s aggregate is not supported on this database",
aggregate=aggregate,
)
)
else:
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
elif expression_type == utils.AdhocMetricExpressionType.SQL:
expression: Any = metric.get("sqlExpression")
if not isinstance(expression, str) or not expression.strip():
-290
View File
@@ -1177,276 +1177,6 @@ def _orderby_modified(
return False
# The frontend emits ``{expressionType: "SQL", sqlExpression: "1 = 0"}`` when
# a native Select filter has "Filter value is required" enabled and no value
# has been selected yet (superset-frontend/src/filters/utils.ts). After
# ``_sanitize_clause`` wraps it in parentheses the resulting ``extras.where``
# clause is ``(1 = 0)``. This is safe — it returns zero rows — and must be
# allowed so that embedded charts are not rejected before the user picks a
# filter value.
_EMPTY_FILTER_SENTINEL = "1 = 0"
def _split_extras_clauses(composed: str) -> list[str]:
"""
Extract raw SQL expressions from a composed ``extras.where`` /
``extras.having`` string.
``_sanitize_clause`` (``form_data_query_context.py:92``) /
``processFilters.ts`` (``superset-ui-core/src/query/processFilters.ts``)
wraps each expression in one layer of parentheses and joins them with
``' AND '``, producing strings like ``(expr1) AND (expr2)``. This
reverses that: split on the ``)\\s+AND\\s+(`` boundary (case-insensitive,
tolerating whitespace variations), strip the outer parens, and return the
raw expressions.
"""
if not composed:
return []
# Unbalanced parens can't be a valid composed clause — fail closed so
# the malformed string lands in the allowed-set check as-is (→ 403)
# instead of splitting into fragments that might individually pass.
if composed.count("(") != composed.count(")"):
return [composed]
raw = re.split(r"\)\s+AND\s+\(", composed, flags=re.IGNORECASE)
# Strip exactly one outer paren added by _sanitize_clause.
if raw[0].startswith("("):
raw[0] = raw[0][1:]
if raw[-1].endswith(")"):
raw[-1] = raw[-1][:-1]
# _sanitize_clause appends ``\n`` inside the parens when the expression
# contains ``--`` (to terminate a trailing line comment). Strip it so
# the result matches the stored raw expression.
return [expr.rstrip("\n") for expr in raw]
def _add_allowed_sql_from_query_context(
extras_allowed: set[str],
col_allowed: set[str],
stored_query_context: dict[str, Any],
) -> None:
"""Add allowed SQL expressions from a stored query context."""
for query in stored_query_context.get("queries") or []:
for param in ("where", "having"):
composed = (query.get("extras") or {}).get(param, "")
for expr in _split_extras_clauses(composed):
extras_allowed.add(expr)
# Keep the full composed value as a fallback in case a stored
# expression contains a literal ") AND (" that the split would
# incorrectly break apart.
if composed:
extras_allowed.add(composed)
for key in ("columns", "groupby"):
for col in query.get(key) or []:
if isinstance(col, dict) and col.get("sqlExpression"):
col_allowed.add(col["sqlExpression"])
def _collect_allowed_sql(
stored_chart: "Slice",
stored_query_context: Optional[dict[str, Any]],
) -> tuple[set[str], set[str]]:
"""
Collect the SQL expressions a guest user is allowed to send.
Returns ``(extras_allowed, col_allowed)``:
* ``extras_allowed`` for validating ``extras.where``/``extras.having``:
adhoc-filter SQL, legacy ``where`` param, stored query-context extras,
and the ``1 = 0`` empty-filter sentinel.
* ``col_allowed`` for validating structured-filter ``col.sqlExpression``:
everything in ``extras_allowed`` plus column SQL expressions from the
chart's dimensions (which cross-filters legitimately reference).
"""
extras_allowed: set[str] = {_EMPTY_FILTER_SENTINEL}
params = stored_chart.params_dict
for flt in params.get("adhoc_filters") or []:
if (
isinstance(flt, dict)
and flt.get("expressionType") == "SQL"
and flt.get("sqlExpression")
):
extras_allowed.add(flt["sqlExpression"])
if params.get("where"):
extras_allowed.add(params["where"])
# Column expressions go only into col_allowed — they must not be
# injectable as WHERE/HAVING predicates.
col_allowed: set[str] = set(extras_allowed)
_add_column_sql_expressions(col_allowed, params)
if stored_query_context:
_add_allowed_sql_from_query_context(
extras_allowed, col_allowed, stored_query_context
)
return extras_allowed, col_allowed
def _add_column_sql_expressions(target: set[str], params: dict[str, Any]) -> None:
"""Add ``sqlExpression`` values from column params to *target*.
Handles both list-valued controls (``columns``, ``groupby``) and
scalar-valued ones (``x_axis``, ``entity``, etc.).
"""
for key in _STORED_COLUMN_PARAMS:
value = params.get(key)
if value is None:
continue
items = value if isinstance(value, (list, tuple)) else [value]
for col in items:
if isinstance(col, dict) and col.get("sqlExpression"):
target.add(col["sqlExpression"])
def _query_has_novel_extras(query: Any, allowed: set[str]) -> bool:
"""Whether a query has novel ``extras.where``/``extras.having`` SQL.
The full composed value is checked first; if it is in ``allowed`` (which
includes full composed values from the stored query context as a fallback)
the split is skipped. If a stored expression contains a literal
``) AND (`` the split may break it into fragments that fail individually
a false positive (403) rather than a bypass, and an acceptable trade-off.
"""
extras = getattr(query, "extras", None) or {}
for param in ("where", "having"):
composed = extras.get(param, "")
if composed and composed not in allowed:
for expr in _split_extras_clauses(composed):
if expr not in allowed:
return True
return False
def _query_has_novel_filter_col(query: Any, allowed: set[str]) -> bool:
"""Whether a query has a structured filter ``col`` not in the allowed set.
Unlike ``_query_has_novel_extras`` this only checks the ``filter[].col``
vector the cross-filter path and intentionally ignores
``extras.where``/``extras.having``. Used for the scoped re-check after
expanding ``allowed`` with sibling dashboard chart expressions: those
borrowed expressions must only legitimize filter columns, not become
injectable as arbitrary WHERE/HAVING predicates.
"""
for flt in getattr(query, "filter", None) or []:
if isinstance(flt, dict):
col = flt.get("col")
if isinstance(col, dict) and col.get("sqlExpression"):
if col["sqlExpression"] not in allowed:
return True
return False
def _add_dashboard_column_expressions(
allowed: set[str], dashboard_id: Any, target_chart_id: int
) -> None:
"""
Add ``sqlExpression`` values from adhoc columns on every chart of the
given dashboard (except the target chart, which is already covered).
This allows cross-filter structured filters whose ``col`` carries the
source chart's custom SQL dimension to pass validation. Called lazily
(only when an unrecognized adhoc SQL col is found) to avoid a DB query
on the common path.
The dashboard is authorized via ``has_guest_access`` and the target chart
must belong to the dashboard; otherwise no expressions are added.
"""
# pylint: disable=import-outside-toplevel
from superset import db, security_manager
from superset.models.dashboard import Dashboard
try:
dashboard_id = int(dashboard_id)
except (TypeError, ValueError):
return
dashboard = (
db.session.query(Dashboard).filter(Dashboard.id == dashboard_id).one_or_none()
)
if dashboard is None:
return
if not security_manager.has_guest_access(dashboard):
return
slice_ids = {s.id for s in dashboard.slices}
if target_chart_id not in slice_ids:
return
for slc in dashboard.slices:
if slc.id == target_chart_id:
continue
_add_column_sql_expressions(allowed, slc.params_dict)
def _sql_filters_modified(
query_context: "QueryContext",
form_data: dict[str, Any],
stored_chart: "Slice",
stored_query_context: Optional[dict[str, Any]],
) -> bool:
"""
Whether the request injects custom SQL not present on the stored chart.
Covers three vectors:
1. ``extras.where`` / ``extras.having`` raw SQL strings.
2. Adhoc filters with ``expressionType == "SQL"`` in ``form_data``.
3. Structured ``{col, op, val}`` filters whose ``col`` carries a
``sqlExpression`` (reaches ``adhoc_column_to_sqla``).
The ``(1 = 0)`` empty-filter sentinel injected by required-but-empty
native Select filters is always allowed. For vector 3, SQL expressions
from all charts on the requesting dashboard are allowed so that
cross-filters referencing a sibling chart's custom SQL dimension pass.
Cache-replay requests (``/data/<cache_key>``) are skipped: the original
request already passed the full check, and ``_sanitize_filters`` may have
rewritten ``extras`` in place before caching (comment normalization,
Jinja rendering), making byte-equality comparison unreliable.
"""
if getattr(query_context, "_from_cache_replay", False) is True:
return False
extras_allowed, col_allowed = _collect_allowed_sql(
stored_chart, stored_query_context
)
# Vector 1: extras.where / extras.having
if any(_query_has_novel_extras(q, extras_allowed) for q in query_context.queries):
return True
# Vector 3: structured filter col with adhoc SQL.
# Sibling chart column expressions (cross-filter) are allowed; the
# dashboard lookup is deferred so the common case pays no DB cost.
if any(_query_has_novel_filter_col(q, col_allowed) for q in query_context.queries):
if dashboard_id := (form_data or {}).get("dashboardId"):
_add_dashboard_column_expressions(
col_allowed, dashboard_id, stored_chart.id
)
if any(
_query_has_novel_filter_col(q, col_allowed) for q in query_context.queries
):
return True
# Vector 2: SQL adhoc filters in form_data
stored_sql_filters: set[str] = {
freeze_value(flt)
for flt in stored_chart.params_dict.get("adhoc_filters") or []
if isinstance(flt, dict) and flt.get("expressionType") == "SQL"
}
for flt in form_data.get("adhoc_filters") or []:
if not isinstance(flt, dict):
continue
if flt.get("expressionType") == "SQL":
if freeze_value(flt) not in stored_sql_filters:
return True
return False
#: Chart params keys that hold the metrics a chart renders. Different chart
#: types store their metrics under control-specific keys (``metric`` for
#: big number, ``x``/``y``/``size`` for bubble, and so on); a guest requesting
@@ -1576,13 +1306,6 @@ def query_context_modified(query_context: "QueryContext") -> bool:
# than accepting any payload, constrain them to the column(s) the dashboard's
# native filter is allowed to target; other chartless paths keep prior
# behavior (see _native_filter_request_modified).
#
# SQL extras (extras.where/having) are NOT validated on chartless paths:
# without a stored chart there is nothing to validate against, and
# tightening this would break legitimate chartless flows (native-filter
# pre-filtering, drill-to-detail) that carry SQL extras. These paths
# are still protected by datasource-access checks in raise_for_access.
# The _sql_filters_modified check below covers chart payloads only.
if stored_chart is None:
return _native_filter_request_modified(query_context)
@@ -1652,19 +1375,6 @@ def query_context_modified(query_context: "QueryContext") -> bool:
)
return True
# SQL predicates (extras.where/having, SQL adhoc filters) must match
# what was saved on the chart; injected custom SQL is rejected.
if _sql_filters_modified(
query_context, form_data, stored_chart, stored_query_context
):
logger.warning(
"Guest chart payload rejected for slice %s: SQL filter/extras "
"not on the stored chart (stored query_context %s)",
stored_chart.id,
stored_context_state,
)
return True
return False
+2 -16
View File
@@ -176,8 +176,6 @@ METRIC_MAP_TYPE = {
"PERCENTILE": "floating",
"VARIANCE": "floating",
"STDDEV": "floating",
"STDDEV_SAMP": "floating",
"VAR_SAMP": "floating",
}
@@ -186,15 +184,6 @@ class AdhocMetricExpressionType(StrEnum):
SQL = "SQL"
# Aggregates with no safe, universal cross-dialect spelling -- unlike
# SUM/COUNT/AVG/MIN/MAX/COUNT_DISTINCT, whose SQL is generated the same way on
# every engine. Support for these is opt-in per `BaseEngineSpec` (see
# `get_extended_aggregation_func`); used to distinguish a genuinely invalid
# aggregate name from one that is valid but unsupported on the current database,
# for a clearer user-facing error.
EXTENDED_METRIC_AGGREGATES = frozenset({"MEDIAN", "STDDEV_SAMP", "VAR_SAMP"})
class SqlExpressionType(StrEnum):
"""Types of SQL expressions that can be validated."""
@@ -1831,14 +1820,11 @@ def get_metric_type_from_column(column: Any, datasource: Explorable) -> str:
expression: str = metric.expression
match = re.match(
r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX|FIRST|LAST"
r"|MEDIAN|STDDEV_SAMP|VAR_SAMP)\s*\((.*)\)",
expression,
re.IGNORECASE,
r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX|FIRST|LAST)\((.*)\)", expression
)
if match:
operation = match.group(1).upper()
operation = match.group(1)
return METRIC_MAP_TYPE.get(operation, "")
logger.debug("Unexpected metric expression type: %s", expression)
-21
View File
@@ -22,7 +22,6 @@ from marshmallow import ValidationError
from pytest_mock import MockerFixture
from superset.charts.schemas import (
ChartDataAdhocMetricSchema,
ChartDataExtrasSchema,
ChartDataPostProcessingOperationSchema,
ChartDataProphetOptionsSchema,
@@ -539,23 +538,3 @@ def test_post_processing_extra_op_is_accepted(app_context: None) -> None:
schema = ChartDataPostProcessingOperationSchema()
assert schema.load({"operation": "_custom_op"})["operation"] == "_custom_op"
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
def test_chart_data_adhoc_metric_schema_accepts_extended_aggregates(
app_context: None, aggregate: str
) -> None:
"""
The chart-data REST schema's ``aggregate`` enum must stay in sync with
``EXTENDED_METRIC_AGGREGATES``, otherwise Swagger/generated clients
reject requests using these compiler-supported aggregates.
"""
schema = ChartDataAdhocMetricSchema()
result = schema.load(
{
"expressionType": "SIMPLE",
"aggregate": aggregate,
"column": {"column_name": "value"},
}
)
assert result["aggregate"] == aggregate
@@ -18,11 +18,15 @@ from unittest.mock import Mock, patch
import pytest
from marshmallow import ValidationError
from pytest_mock import MockerFixture
from superset.commands.dataset.create import CreateDatasetCommand
from superset.commands.dataset.exceptions import DatasetInvalidError
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import SupersetParseError
from superset.exceptions import (
SupersetGenericDBErrorException,
SupersetParseError,
)
from superset.models.core import Database
@@ -250,3 +254,83 @@ def test_create_dataset_generic_exists_error_when_no_twin() -> None:
)
with pytest.raises(DatasetInvalidError):
command.validate()
def test_create_dataset_metadata_fetch_error_is_structured(
mocker: MockerFixture,
) -> None:
"""A metadata-fetch failure must surface the engine's own message.
``run()`` executes the SQL to introspect columns; the resulting
``SupersetGenericDBErrorException`` used to escape as a 500 "Fatal error".
"""
mocker.patch.object(CreateDatasetCommand, "validate")
dataset = Mock()
dataset.fetch_metadata.side_effect = SupersetGenericDBErrorException(
message="Invalid SQL: Unable to parse: SELECT ...",
)
mocker.patch(
"superset.commands.dataset.create.DatasetDAO.create",
return_value=dataset,
)
command = CreateDatasetCommand(
{
"database": 1,
"table_name": "dataset wrong",
"sql": "SELECT ...",
}
)
with pytest.raises(DatasetInvalidError) as exc_info:
command.run()
validation_errors = exc_info.value._exceptions
assert len(validation_errors) == 1
assert validation_errors[0].field_name == "sql"
assert "Invalid SQL: Unable to parse: SELECT ..." in str(
validation_errors[0].messages[0]
)
def test_create_dataset_metadata_fetch_error_physical_table(
mocker: MockerFixture,
) -> None:
"""The same conversion applies to physical datasets, keyed on ``table``."""
mocker.patch.object(CreateDatasetCommand, "validate")
dataset = Mock()
dataset.fetch_metadata.side_effect = SupersetGenericDBErrorException(
message="(psycopg2.OperationalError) could not connect to server",
)
mocker.patch(
"superset.commands.dataset.create.DatasetDAO.create",
return_value=dataset,
)
command = CreateDatasetCommand({"database": 1, "table_name": "physical_table"})
with pytest.raises(DatasetInvalidError) as exc_info:
command.run()
validation_errors = exc_info.value._exceptions
assert validation_errors[0].field_name == "table"
assert "could not connect to server" in str(validation_errors[0].messages[0])
def test_create_dataset_run_succeeds_when_metadata_fetch_works(
mocker: MockerFixture,
) -> None:
"""Control: the happy path still returns the created dataset."""
mocker.patch.object(CreateDatasetCommand, "validate")
dataset = Mock()
mocker.patch(
"superset.commands.dataset.create.DatasetDAO.create",
return_value=dataset,
)
command = CreateDatasetCommand(
{"database": 1, "table_name": "good_dataset", "sql": "SELECT 1 AS a"}
)
assert command.run() is dataset
dataset.fetch_metadata.assert_called_once()
+43
View File
@@ -214,3 +214,46 @@ def test_handle_filters_args_returns_request_scoped_filters(
fresh_filters = api.datamodel.get_filters.return_value
assert fresh_filters.rest_add_filters.call_count == 2
assert fresh_filters.get_joined_filters.call_count == 2
def test_post_dataset_with_invalid_sql_returns_actionable_422(
session: Session,
client: Any,
full_api_access: None,
) -> None:
"""Saving a dataset over unrunnable SQL must explain what is wrong.
With blanket database access ``validate()`` never parses the SQL, so
``run()``'s column introspection is the first thing to reject it. That
used to surface as a bare 500 ``{"message": "Fatal error"}``.
"""
from superset.connectors.sqla.models import SqlaTable
from superset.models.core import Database
SqlaTable.metadata.create_all(db.session.get_bind())
database = Database(database_name="invalid_sql_db", sqlalchemy_uri="sqlite://")
db.session.add(database)
db.session.flush()
response = client.post(
"/api/v1/dataset/",
json={
"database": database.id,
"schema": "main",
"table_name": "dataset wrong",
"sql": "SELECT ...",
},
)
assert response.status_code == 422
message = response.json["message"]
assert "Fatal error" not in str(message)
# Not the parser's exact wording -- that would break on a sqlglot bump.
assert message["sql"][0].startswith("Invalid SQL")
# The failed create must not leave a half-built dataset behind.
assert (
db.session.query(SqlaTable).filter_by(table_name="dataset wrong").one_or_none()
is None
)
@@ -301,43 +301,6 @@ def test_aurora_mysql_update_params_from_encrypted_extra_with_iam() -> None:
# SSL should be configured via the database's extra settings.
def test_aurora_postgres_inherits_extended_aggregations() -> None:
"""
Aurora PostgreSQL (and its Data API variant) is AWS's wire- and
SQL-compatible managed Postgres, not a forked query engine, so it
inherits `_extended_aggregations` from `PostgresBaseEngineSpec`
unmodified -- see the comment above that dict.
"""
from superset.db_engine_specs.aurora import (
AuroraPostgresDataAPI,
AuroraPostgresEngineSpec,
)
for spec in (AuroraPostgresEngineSpec, AuroraPostgresDataAPI):
assert spec.get_extended_aggregation_func("MEDIAN") is not None
assert spec.get_extended_aggregation_func("STDDEV_SAMP") is not None
assert spec.get_extended_aggregation_func("VAR_SAMP") is not None
def test_aurora_mysql_inherits_extended_aggregations() -> None:
"""
Aurora MySQL (and its Data API variant) is AWS's wire- and
SQL-compatible managed MySQL, not a forked query engine, so it inherits
`_extended_aggregations` from `MySQLEngineSpec` unmodified -- see the
comment above that dict.
"""
from superset.db_engine_specs.aurora import (
AuroraMySQLDataAPI,
AuroraMySQLEngineSpec,
)
for spec in (AuroraMySQLEngineSpec, AuroraMySQLDataAPI):
assert spec.get_extended_aggregation_func("STDDEV_SAMP") is not None
assert spec.get_extended_aggregation_func("VAR_SAMP") is not None
# Same as MySQL, MEDIAN is not supported.
assert spec.get_extended_aggregation_func("MEDIAN") is None
def test_aurora_data_api_classes_unchanged() -> None:
from superset.db_engine_specs.aurora import (
AuroraMySQLDataAPI,
@@ -1508,22 +1508,3 @@ def test_array_capabilities_raise_when_unsupported(method: str) -> None:
args = (column("c"), ["v"]) if "contains" in method else (column("c"),)
with pytest.raises(NotImplementedError):
fn(*args)
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
def test_base_spec_extended_aggregation_func_defaults_to_unsupported(
aggregate: str,
) -> None:
"""
By default, an engine spec has no verified expression for the "extended"
aggregates (MEDIAN/STDDEV_SAMP/VAR_SAMP) -- they must be explicitly opted
into per engine spec, the same way `supports_grouping_sets` and
`_time_grain_expressions` work. Silence here means "unsupported", not
"untested" -- callers must not fall back to guessing SQL.
"""
assert BaseEngineSpec.get_extended_aggregation_func(aggregate) is None
def test_base_spec_extended_aggregation_func_unknown_name_is_unsupported() -> None:
"""An aggregate name outside the known extended set is also just None."""
assert BaseEngineSpec.get_extended_aggregation_func("NOT_A_REAL_AGGREGATE") is None
@@ -191,62 +191,3 @@ def test_fetch_data_preserves_cursor_description(mocker: MockerFixture) -> None:
assert [col[0] for col in cursor.description] == ["col1", "col2"]
finally:
raw_conn.close()
def test_extended_aggregation_func_median_stddev_var_compiles() -> None:
"""
MEDIAN/STDDEV_SAMP/VAR_SAMP compile to the expected DuckDB SQL function
calls. See `test_extended_aggregation_func_median_stddev_var_executes`
for verification against a live in-process DuckDB instance.
"""
from sqlalchemy import column
from superset.db_engine_specs.duckdb import DuckDBEngineSpec
col = column("sales")
for aggregate, expected_sql in [
("MEDIAN", "median(sales)"),
("STDDEV_SAMP", "stddev_samp(sales)"),
("VAR_SAMP", "var_samp(sales)"),
]:
func = DuckDBEngineSpec.get_extended_aggregation_func(aggregate)
assert func is not None
compiled = str(func(col).compile(compile_kwargs={"literal_binds": True}))
assert compiled == expected_sql
def test_extended_aggregation_func_median_stddev_var_executes() -> None:
"""
MEDIAN/STDDEV_SAMP/VAR_SAMP execute against a live in-process DuckDB
instance and return values matching Python's `statistics` module
(sample standard deviation/variance) for the same input.
"""
import statistics
from sqlalchemy import create_engine, literal_column, select, text
from superset.db_engine_specs.duckdb import DuckDBEngineSpec
values = [1.0, 2.0, 4.0, 8.0, 16.0]
engine = create_engine("duckdb:///:memory:")
with engine.connect() as conn:
conn.execute(text("CREATE TABLE t (sales DOUBLE)"))
conn.execute(
text("INSERT INTO t VALUES (:sales)"),
[{"sales": v} for v in values],
)
expected = {
"MEDIAN": statistics.median(values),
"STDDEV_SAMP": statistics.stdev(values),
"VAR_SAMP": statistics.variance(values),
}
for aggregate, expected_value in expected.items():
func = DuckDBEngineSpec.get_extended_aggregation_func(aggregate)
assert func is not None
query = select(func(literal_column("sales"))).select_from(text("t"))
result = conn.execute(query).scalar()
assert result == pytest.approx(expected_value)
@@ -1,68 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
`_extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is verified against
real engine behavior before being enabled -- see `BaseEngineSpec` for the
rationale. `PostgresBaseEngineSpec`/`PostgresEngineSpec`/`MySQLEngineSpec`
subclasses that share SQL dialect helpers with Postgres/MySQL but run a
materially different query engine (a proprietary appliance, a distributed
SQL engine, or an OLAP engine) must not silently inherit that dict: each
verified engine spec opts in explicitly, everything else stays unsupported
until someone verifies it against a live instance.
"""
from typing import Type
import pytest
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.cockroachdb import CockroachDbEngineSpec
from superset.db_engine_specs.doris import DorisEngineSpec
from superset.db_engine_specs.greenplum import GreenplumEngineSpec
from superset.db_engine_specs.hana import HanaEngineSpec
from superset.db_engine_specs.hologres import HologresEngineSpec
from superset.db_engine_specs.netezza import NetezzaEngineSpec
from superset.db_engine_specs.oceanbase import OceanBaseEngineSpec
from superset.db_engine_specs.risingwave import RisingWaveDbEngineSpec
from superset.db_engine_specs.snowflake import SnowflakeEngineSpec
from superset.db_engine_specs.starrocks import StarRocksEngineSpec
from superset.db_engine_specs.vertica import VerticaEngineSpec
from superset.db_engine_specs.yugabytedb import YugabyteDBEngineSpec
@pytest.mark.parametrize(
"spec_cls",
[
VerticaEngineSpec,
NetezzaEngineSpec,
HanaEngineSpec,
SnowflakeEngineSpec,
CockroachDbEngineSpec,
GreenplumEngineSpec,
RisingWaveDbEngineSpec,
YugabyteDBEngineSpec,
HologresEngineSpec,
DorisEngineSpec,
StarRocksEngineSpec,
OceanBaseEngineSpec,
],
)
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
def test_unverified_postgres_and_mysql_family_specs_reject_extended_aggregations(
spec_cls: Type[BaseEngineSpec], aggregate: str
) -> None:
assert spec_cls.get_extended_aggregation_func(aggregate) is None
@@ -1,35 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.db_engine_specs.mariadb import MariaDBEngineSpec
from superset.db_engine_specs.mysql import MySQLEngineSpec
def test_mariadb_inherits_from_mysql() -> None:
assert issubclass(MariaDBEngineSpec, MySQLEngineSpec)
def test_mariadb_inherits_extended_aggregations() -> None:
"""
MariaDB is a MySQL fork implementing the same aggregate functions, not a
materially different query engine, so it inherits `_extended_aggregations`
from `MySQLEngineSpec` unmodified -- see the comment above that dict.
"""
assert MariaDBEngineSpec.get_extended_aggregation_func("STDDEV_SAMP") is not None
assert MariaDBEngineSpec.get_extended_aggregation_func("VAR_SAMP") is not None
# Same as MySQL, MEDIAN is not supported.
assert MariaDBEngineSpec.get_extended_aggregation_func("MEDIAN") is None
@@ -401,39 +401,3 @@ def test_identifier_quote_uses_backticks() -> None:
"end": "`",
"escape_by_doubling": True,
}
def test_extended_aggregation_func_stddev_var_sample() -> None:
"""
Verified against a live mysql:8.0 instance, including under GROUP BY ...
WITH ROLLUP: these produce the correct database-wide sample statistic, and
match the same-input results from postgres/duckdb exactly.
"""
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
col = column("sales")
stddev_expr = spec.get_extended_aggregation_func("STDDEV_SAMP")
assert stddev_expr is not None
assert (
str(stddev_expr(col).compile(compile_kwargs={"literal_binds": True}))
== "stddev_samp(sales)"
)
var_expr = spec.get_extended_aggregation_func("VAR_SAMP")
assert var_expr is not None
assert (
str(var_expr(col).compile(compile_kwargs={"literal_binds": True}))
== "var_samp(sales)"
)
def test_extended_aggregation_func_median_unsupported() -> None:
"""
MySQL has neither a MEDIAN function nor PERCENTILE_CONT (confirmed against
a live mysql:8.0 instance: both error). Must not silently fall back to
a guessed expression.
"""
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
assert spec.get_extended_aggregation_func("MEDIAN") is None
@@ -501,30 +501,3 @@ def test_get_schema_names_excludes_only_actual_system_schemas(
"pgstats",
"information_schema",
}
@pytest.mark.parametrize(
("aggregate", "expected_sql"),
[
("MEDIAN", "percentile_cont(0.5) WITHIN GROUP (ORDER BY sales)"),
("STDDEV_SAMP", "stddev_samp(sales)"),
("VAR_SAMP", "var_samp(sales)"),
],
)
def test_extended_aggregation_func_compiles_expected_sql(
aggregate: str, expected_sql: str
) -> None:
"""
Verified against a live postgres:16 instance (including under GROUPING
SETS): these expressions compute the correct database-wide statistic, not
an aggregate-of-per-group-aggregates.
"""
func = spec.get_extended_aggregation_func(aggregate)
assert func is not None
compiled = str(
func(column("sales")).compile(
dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}
)
)
assert compiled == expected_sql
@@ -79,33 +79,3 @@ def test_normalize_table_name_for_upload(
assert normalized_table == expected_table
assert normalized_schema == expected_schema
def test_extended_aggregation_func_inherited_from_postgres() -> None:
"""
Redshift is a Postgres fork and inherits STDDEV_SAMP/VAR_SAMP from
`PostgresBaseEngineSpec` -- its documented SQL function reference matches
Postgres for these functions, though this has not been separately
verified against a live Redshift instance (see the SIP doc for caveats).
MEDIAN is overridden rather than inherited (see
`RedshiftEngineSpec._extended_aggregations`): unlike Postgres, Redshift
documents a native `MEDIAN(x)` function, so Redshift emits that directly
instead of Postgres's `percentile_cont(0.5) WITHIN GROUP (ORDER BY col)`
spelling -- which also sidesteps Redshift's documented restriction
against more than one sort-based aggregate with a different ORDER BY in
the same query, e.g. `MEDIAN(sales)` alongside `MEDIAN(margin)`.
"""
from sqlalchemy import column
col = column("sales")
median_func = RedshiftEngineSpec.get_extended_aggregation_func("MEDIAN")
assert median_func is not None
assert (
str(median_func(col).compile(compile_kwargs={"literal_binds": True}))
== "median(sales)"
)
for aggregate in ("STDDEV_SAMP", "VAR_SAMP"):
assert RedshiftEngineSpec.get_extended_aggregation_func(aggregate) is not None
@@ -1,37 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
from superset.db_engine_specs.timescaledb import TimescaleDBEngineSpec
def test_timescaledb_inherits_from_postgres_base() -> None:
assert issubclass(TimescaleDBEngineSpec, PostgresBaseEngineSpec)
def test_timescaledb_inherits_extended_aggregations() -> None:
"""
TimescaleDB is a Postgres extension, not a forked query engine -- it runs
unmodified Postgres aggregate execution -- so it inherits
`_extended_aggregations` from `PostgresBaseEngineSpec` unmodified -- see
the comment above that dict.
"""
assert TimescaleDBEngineSpec.get_extended_aggregation_func("MEDIAN") is not None
assert (
TimescaleDBEngineSpec.get_extended_aggregation_func("STDDEV_SAMP") is not None
)
assert TimescaleDBEngineSpec.get_extended_aggregation_func("VAR_SAMP") is not None
@@ -89,24 +89,6 @@ class TestCreateMetricObject:
assert result["optionName"] == "metric_revenue"
assert result["expressionType"] == "SIMPLE"
def test_create_metric_object_normalizes_stddev_var_shorthand(self) -> None:
"""
Pre-SIP "STDDEV"/"VAR" shorthand is normalized to the real,
unambiguous aggregate names ("STDDEV_SAMP"/"VAR_SAMP") this chart
can actually query -- see docs/sip/median-stddev-variance-aggregates.md.
Before this, a chart built with "STDDEV"/"VAR" always errored at
query time, since that name was never a real Superset aggregate.
"""
stddev_col = ColumnRef(name="price", aggregate="STDDEV", label="Std Dev")
result = create_metric_object(stddev_col)
assert isinstance(result, dict)
assert result["aggregate"] == "STDDEV_SAMP"
var_col = ColumnRef(name="price", aggregate="VAR", label="Variance")
result = create_metric_object(var_col)
assert isinstance(result, dict)
assert result["aggregate"] == "VAR_SAMP"
def test_create_metric_object_default_aggregate(self) -> None:
"""Test creating metric object with default aggregate"""
col = ColumnRef(name="orders")
-70
View File
@@ -2130,76 +2130,6 @@ def test_adhoc_metric_to_sqla_invalid_simple_aggregate_raises_validation_error(
table.adhoc_metric_to_sqla(metric, {})
@pytest.mark.parametrize(
"aggregate,expected_substring",
[
("MEDIAN", "percentile_cont"),
("STDDEV_SAMP", "stddev_samp"),
("VAR_SAMP", "var_samp"),
],
)
def test_adhoc_metric_to_sqla_extended_aggregate_on_supported_engine(
aggregate: str,
expected_substring: str,
) -> None:
"""
MEDIAN/STDDEV_SAMP/VAR_SAMP compile correctly end-to-end on an engine that
supports them (Postgres), via the same `adhoc_metric_to_sqla` path every
other aggregate uses -- no pivot-table-specific code involved.
"""
from superset.connectors.sqla.models import SqlaTable, TableColumn
from superset.models.core import Database
pg_database = Database(database_name="pg", sqlalchemy_uri="postgresql://u:p@h/d")
table = SqlaTable(
database=pg_database,
schema=None,
table_name="t",
columns=[TableColumn(column_name="sales")],
)
metric: AdhocMetric = {
"expressionType": "SIMPLE",
"column": {"column_name": "sales"},
"aggregate": aggregate,
"label": f"{aggregate} sales",
}
sqla_metric = table.adhoc_metric_to_sqla(metric, {})
assert expected_substring in str(sqla_metric).lower()
def test_adhoc_metric_to_sqla_extended_aggregate_on_unsupported_engine_raises_specific_error( # noqa: E501
database: Database,
) -> None:
"""
A recognized extended aggregate (MEDIAN) that this engine (SQLite, via the
`database` fixture) has no verified expression for raises a specific
"not supported on this database" error, distinct from the generic
"invalid aggregate" error a bogus aggregate name gets -- callers should be
able to tell "this isn't a real thing" from "this engine can't do it"
without emitting unverified SQL either way.
"""
from superset.connectors.sqla.models import SqlaTable, TableColumn
from superset.exceptions import QueryObjectValidationError
table = SqlaTable(
database=database,
schema=None,
table_name="t",
columns=[TableColumn(column_name="a")],
)
metric: AdhocMetric = {
"expressionType": "SIMPLE",
"column": {"column_name": "a"},
"aggregate": "MEDIAN",
"label": "Median a",
}
with pytest.raises(QueryObjectValidationError, match="not supported"):
table.adhoc_metric_to_sqla(metric, {})
@pytest.mark.parametrize("sql_expression", [None, "", " "])
def test_adhoc_metric_to_sqla_invalid_sql_expression_raises_validation_error(
database: Database,
-681
View File
@@ -36,7 +36,6 @@ from superset.extensions import appbuilder
from superset.models.slice import Slice
from superset.security.manager import (
_collect_sortable_identifiers,
_sql_filters_modified,
freeze_value,
query_context_modified,
SupersetSecurityManager,
@@ -3792,683 +3791,3 @@ def test_validate_guest_token_resources_accepts_embedded_int_id(
sm.validate_guest_token_resources(
[{"type": GuestTokenResourceType.DASHBOARD, "id": 5}]
)
# ---------------------------------------------------------------------------
# _sql_filters_modified block custom SQL injection by guest users
# ---------------------------------------------------------------------------
def test_sql_filters_extras_where_injected_blocked(
mocker: MockerFixture,
) -> None:
"""Injecting extras.where when the chart has no SQL filters is blocked."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"metrics": ["count"]}
query = QueryObject(extras={"where": "1=1"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_extras_having_injected_blocked(
mocker: MockerFixture,
) -> None:
"""Injecting extras.having when the chart has no SQL filters is blocked."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(extras={"having": "COUNT(*) > 0"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_extras_where_replay_allowed(
mocker: MockerFixture,
) -> None:
"""Replaying the chart's own SQL WHERE filter is allowed."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "region = 'EMEA'",
"clause": "WHERE",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
# freeform_where_having wraps each clause in parens
query = QueryObject(extras={"where": "(region = 'EMEA')"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_extras_having_replay_allowed(
mocker: MockerFixture,
) -> None:
"""Replaying the chart's own SQL HAVING filter is allowed."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "SUM(sales) > 100",
"clause": "HAVING",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
query = QueryObject(extras={"having": "(SUM(sales) > 100)"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_adhoc_sql_filter_injected_blocked(
mocker: MockerFixture,
) -> None:
"""Injecting a new SQL adhoc filter not on the stored chart is blocked."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject()
query_context.queries = [query]
injected_filter = {
"expressionType": "SQL",
"sqlExpression": "1=1",
"clause": "WHERE",
}
form_data: dict[str, Any] = {"slice_id": 1, "adhoc_filters": [injected_filter]}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_adhoc_sql_filter_replay_allowed(
mocker: MockerFixture,
) -> None:
"""Replaying the exact stored SQL adhoc filter is allowed."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "region = 'EMEA'",
"clause": "WHERE",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
query = QueryObject()
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1, "adhoc_filters": [sql_filter]}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_empty_extras_always_allowed(
mocker: MockerFixture,
) -> None:
"""No SQL in extras is always allowed, even when the chart has SQL filters."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "region = 'EMEA'",
"clause": "WHERE",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
query = QueryObject()
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_from_stored_qc_allowed(
mocker: MockerFixture,
) -> None:
"""extras.where from stored query_context is allowed."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
stored_qc = {
"queries": [{"extras": {"where": "(col > 5)"}}],
}
query = QueryObject(extras={"where": "(col > 5)"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
def test_sql_filters_multi_query_stored_predicate_allowed(
mocker: MockerFixture,
) -> None:
"""Multiple queries replaying predicates from the stored chart are allowed.
The allowed set is global across all stored queries per-query pinning is
intentionally not applied because there is no stable identity linking a
request query to a stored query, and all queries share the same
chart/datasource so predicates only restrict rows, never expand access.
"""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
stored_qc = {
"queries": [
{"extras": {"where": "(region = 'EMEA')"}},
{"extras": {"where": "(status = 'active')"}},
],
}
# Both request queries use predicates from the stored chart.
query_context.queries = [
QueryObject(extras={"where": "(region = 'EMEA')"}),
QueryObject(extras={"where": "(status = 'active')"}),
]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
def test_sql_filters_multi_query_novel_predicate_blocked(
mocker: MockerFixture,
) -> None:
"""A novel predicate on any query is blocked even when others are valid."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
stored_qc = {
"queries": [{"extras": {"where": "(region = 'EMEA')"}}],
}
query_context.queries = [
QueryObject(extras={"where": "(region = 'EMEA')"}),
QueryObject(extras={"where": "(1=1)"}), # not stored
]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
def test_sql_filters_different_sql_blocked(
mocker: MockerFixture,
) -> None:
"""Modified SQL (appending extra predicates) is blocked."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "col > 5",
"clause": "WHERE",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
# Attacker appends extra predicate
query = QueryObject(
extras={"where": "(col > 5) AND (1=1)"},
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_simple_filters_not_blocked(
mocker: MockerFixture,
) -> None:
"""SIMPLE structured filters (from dashboard native filters) are not blocked."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(
filters=[{"col": "country", "op": "==", "val": "US"}],
)
query_context.queries = [query]
simple_adhoc_filter = {
"expressionType": "SIMPLE",
"subject": "country",
"operator": "==",
"comparator": "US",
"clause": "WHERE",
}
form_data: dict[str, Any] = {
"slice_id": 1,
"adhoc_filters": [simple_adhoc_filter],
}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_structured_filter_adhoc_col_blocked(
mocker: MockerFixture,
) -> None:
"""Structured filter with an adhoc SQL column in ``col`` is blocked.
``ChartDataFilterSchema.col`` is ``fields.Raw``, so an attacker can pass
an adhoc column dict that reaches ``adhoc_column_to_sqla`` and executes
arbitrary SQL in the WHERE clause.
"""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
adhoc_col: Any = {
"expressionType": "SQL",
"sqlExpression": "1; DROP TABLE users--",
"label": "x",
}
query = QueryObject(
filters=[{"col": adhoc_col, "op": "!=", "val": "z"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_structured_filter_stored_adhoc_col_allowed(
mocker: MockerFixture,
) -> None:
"""Cross-filter with an adhoc SQL column matching a stored chart dimension
is allowed."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {
"columns": [
{"sqlExpression": "YEAR(order_date)", "label": "order_year"},
],
}
adhoc_col: Any = {
"sqlExpression": "YEAR(order_date)",
"label": "order_year",
}
query = QueryObject(
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_cross_filter_adhoc_col_from_sibling_chart_allowed(
mocker: MockerFixture,
) -> None:
"""Cross-filter with an adhoc SQL column from a sibling chart on the same
dashboard is allowed."""
from superset.models.dashboard import Dashboard
# Target chart (chart B) has no custom SQL columns.
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.id = 2
stored_chart.params_dict = {"metrics": ["count"]}
# Source chart (chart A) has the custom SQL dimension.
sibling_chart = mocker.MagicMock()
sibling_chart.id = 1
sibling_chart.params_dict = {
"columns": [
{"sqlExpression": "YEAR(order_date)", "label": "order_year"},
],
}
# Dashboard contains both charts.
dashboard = mocker.MagicMock(spec=Dashboard)
dashboard.slices = [sibling_chart, stored_chart]
mocker.patch("superset.db.session.query")
db_query = mocker.patch("superset.db.session.query").return_value
db_query.filter.return_value.one_or_none.return_value = dashboard
mocker.patch(
"superset.security_manager.has_guest_access",
return_value=True,
)
adhoc_col: Any = {
"sqlExpression": "YEAR(order_date)",
"label": "order_year",
}
query = QueryObject(
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 10}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_cross_filter_rejected_for_unauthorized_dashboard(
mocker: MockerFixture,
) -> None:
"""Cross-filter lookup must not use a dashboard the guest has no access to."""
from superset.models.dashboard import Dashboard
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.id = 2
stored_chart.params_dict = {}
sibling_chart = mocker.MagicMock()
sibling_chart.id = 1
sibling_chart.params_dict = {
"columns": [{"sqlExpression": "YEAR(order_date)", "label": "order_year"}],
}
dashboard = mocker.MagicMock(spec=Dashboard)
dashboard.slices = [sibling_chart, stored_chart]
mocker.patch("superset.db.session.query")
db_query = mocker.patch("superset.db.session.query").return_value
db_query.filter.return_value.one_or_none.return_value = dashboard
mocker.patch(
"superset.security_manager.has_guest_access",
return_value=False,
)
adhoc_col: Any = {"sqlExpression": "YEAR(order_date)", "label": "order_year"}
query = QueryObject(
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 999}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_cross_filter_rejected_when_chart_not_on_dashboard(
mocker: MockerFixture,
) -> None:
"""Cross-filter lookup must verify the target chart belongs to the dashboard."""
from superset.models.dashboard import Dashboard
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.id = 99 # not on the dashboard
stored_chart.params_dict = {}
sibling_chart = mocker.MagicMock()
sibling_chart.id = 1
sibling_chart.params_dict = {
"columns": [{"sqlExpression": "YEAR(order_date)", "label": "order_year"}],
}
dashboard = mocker.MagicMock(spec=Dashboard)
dashboard.slices = [sibling_chart] # stored_chart not here
mocker.patch("superset.db.session.query")
db_query = mocker.patch("superset.db.session.query").return_value
db_query.filter.return_value.one_or_none.return_value = dashboard
mocker.patch(
"superset.security_manager.has_guest_access",
return_value=True,
)
adhoc_col: Any = {"sqlExpression": "YEAR(order_date)", "label": "order_year"}
query = QueryObject(
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 99, "dashboardId": 10}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_sibling_expressions_cannot_inject_where_having(
mocker: MockerFixture,
) -> None:
"""Sibling chart column expressions must not legitimize novel WHERE/HAVING."""
from superset.models.dashboard import Dashboard
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.id = 2
stored_chart.params_dict = {}
# Sibling has a column expression that an attacker tries to use as WHERE.
sibling_chart = mocker.MagicMock()
sibling_chart.id = 1
sibling_chart.params_dict = {
"columns": [
{"sqlExpression": "(SELECT secret FROM users LIMIT 1)", "label": "x"},
],
}
dashboard = mocker.MagicMock(spec=Dashboard)
dashboard.slices = [sibling_chart, stored_chart]
mocker.patch("superset.db.session.query")
db_query = mocker.patch("superset.db.session.query").return_value
db_query.filter.return_value.one_or_none.return_value = dashboard
# Attacker injects the sibling expression into extras.where.
query = QueryObject(
extras={"where": "(SELECT secret FROM users LIMIT 1)"},
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 10}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_collect_allowed_sql_includes_scalar_column_params(
mocker: MockerFixture,
) -> None:
"""Scalar column params like x_axis contribute their sqlExpression."""
from superset.security.manager import _collect_allowed_sql
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {
"x_axis": {"sqlExpression": "DATE_TRUNC('month', ts)", "label": "m"},
"groupby": [{"sqlExpression": "UPPER(country)", "label": "c"}],
}
_, col_allowed = _collect_allowed_sql(stored_chart, None)
assert "DATE_TRUNC('month', ts)" in col_allowed
assert "UPPER(country)" in col_allowed
def test_sql_filters_structured_filter_string_col_allowed(
mocker: MockerFixture,
) -> None:
"""Structured filter with a plain string column is allowed."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(
filters=[{"col": "status", "op": "==", "val": "active"}],
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_empty_filter_sentinel_allowed(
mocker: MockerFixture,
) -> None:
"""The ``(1 = 0)`` sentinel from a required-but-empty native filter is allowed."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(extras={"where": "(1 = 0)"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_double_sentinel_allowed(
mocker: MockerFixture,
) -> None:
"""Two required-but-empty filters compose ``(1 = 0) AND (1 = 0)``."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(extras={"where": "(1 = 0) AND (1 = 0)"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_stored_clause_plus_sentinel_allowed(
mocker: MockerFixture,
) -> None:
"""A stored SQL filter composed with the empty-filter sentinel is allowed."""
sql_filter = {
"expressionType": "SQL",
"sqlExpression": "region = 'EMEA'",
"clause": "WHERE",
}
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
query = QueryObject(
extras={"where": "(region = 'EMEA') AND (1 = 0)"},
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_non_dict_adhoc_filter_skipped(
mocker: MockerFixture,
) -> None:
"""Non-dict items in adhoc_filters are skipped, not 500."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject()
query_context.queries = [query]
form_data: dict[str, Any] = {
"slice_id": 1,
"adhoc_filters": ["not_a_dict", 42, None],
}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_raise_for_access_guest_user_sql_filter_injection_blocked(
mocker: MockerFixture,
app_context: None,
stored_metrics: list[AdhocMetric],
) -> None:
"""Guest user injecting SQL via extras.where is rejected by raise_for_access."""
sm = SupersetSecurityManager(appbuilder)
mocker.patch.object(sm, "is_guest_user", return_value=True)
mocker.patch.object(sm, "can_access", return_value=True)
query_context = mocker.MagicMock()
query_context.slice_.id = 42
query_context.slice_.query_context = None
query_context.slice_.params_dict = {"metrics": stored_metrics}
query_context.form_data = {"slice_id": 42, "metrics": stored_metrics}
query_context.queries = [
QueryObject(
metrics=stored_metrics, # type: ignore
extras={"where": "1=1 UNION SELECT password FROM users"},
)
]
with pytest.raises(SupersetSecurityException):
sm.raise_for_access(query_context=query_context)
def test_sql_filters_cache_replay_skips_check(
mocker: MockerFixture,
) -> None:
"""Cache-replay requests skip the SQL filter check."""
query_context = mocker.MagicMock()
query_context._from_cache_replay = True
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(extras={"where": "(injected SQL)"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_column_expression_cannot_become_where(
mocker: MockerFixture,
) -> None:
"""A chart's column sqlExpression must not be injectable as extras.where."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {
"columns": [
{
"sqlExpression": "(SELECT secret FROM users LIMIT 1)",
"label": "x",
},
],
}
query = QueryObject(
extras={"where": "((SELECT secret FROM users LIMIT 1))"},
)
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
def test_sql_filters_unbalanced_parens_rejected(
mocker: MockerFixture,
) -> None:
"""Unbalanced parens in extras.where are rejected (403, not 500)."""
query_context = mocker.MagicMock()
stored_chart = mocker.MagicMock()
stored_chart.params_dict = {}
query = QueryObject(extras={"where": "(a) AND (b"})
query_context.queries = [query]
form_data: dict[str, Any] = {"slice_id": 1}
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
-15
View File
@@ -38,21 +38,6 @@ def test_column_with_valid_operation():
assert (get_metric_type_from_column(column, datasource)) == "floating"
@pytest.mark.parametrize(
"expression",
[
"stddev_samp(my_column)",
"STDDEV_SAMP (my_column)",
"StdDev_Samp(my_column)",
],
)
def test_column_with_lowercase_or_whitespaced_operation(expression):
metric = SqlMetric(metric_name="my_column", expression=expression)
datasource = MagicMock(metrics=[metric])
column = "my_column"
assert (get_metric_type_from_column(column, datasource)) == "floating"
def test_column_with_invalid_operation():
metric = SqlMetric(metric_name="my_column", expression="INVALID(my_column)")
datasource = MagicMock(metrics=[metric])