mirror of
https://github.com/apache/superset.git
synced 2026-08-21 15:41:16 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a6c1b977b | ||
|
|
e8577368d3 | ||
|
|
540f8cb2d0 | ||
|
|
aae997e546 |
@@ -78,14 +78,14 @@ jobs:
|
||||
USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true' || 'false' }}
|
||||
services:
|
||||
postgres:
|
||||
image: ghcr.io/apache/superset/ci/postgres:17-alpine
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
@@ -186,14 +186,14 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
services:
|
||||
postgres:
|
||||
image: ghcr.io/apache/superset/ci/postgres:17-alpine
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
|
||||
@@ -53,7 +53,9 @@ jobs:
|
||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||
services:
|
||||
mysql:
|
||||
image: ghcr.io/apache/superset/ci/mysql:8.0
|
||||
image: mysql:8.0
|
||||
# Authenticated pulls use our higher Docker Hub rate limit. Empty on
|
||||
# fork PRs (secrets unavailable) -> runner falls back to anonymous.
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
ports:
|
||||
@@ -64,7 +66,7 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
options: --entrypoint redis-server
|
||||
ports:
|
||||
- 16379:6379
|
||||
@@ -141,7 +143,7 @@ jobs:
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: ghcr.io/apache/superset/ci/postgres:17-alpine
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -150,7 +152,7 @@ jobs:
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
@@ -200,7 +202,7 @@ jobs:
|
||||
sqlite:///${{ github.workspace }}/.temp/examples.db?check_same_thread=true
|
||||
services:
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
|
||||
services:
|
||||
postgres:
|
||||
image: ghcr.io/apache/superset/ci/postgres:17-alpine
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
presto:
|
||||
image: ghcr.io/apache/superset/ci/presto:350-e.6
|
||||
image: starburstdata/presto:350-e.6
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
# GitHub action runner's default installations
|
||||
- 15433:8080
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
UPLOAD_FOLDER: /tmp/.superset/uploads/
|
||||
services:
|
||||
postgres:
|
||||
image: ghcr.io/apache/superset/ci/postgres:17-alpine
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: ghcr.io/apache/superset/ci/redis:7-alpine
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
|
||||
@@ -304,7 +304,7 @@ pre-commit run eslint # Frontend linting
|
||||
## Platform-Specific Instructions
|
||||
|
||||
- **[CLAUDE.md](CLAUDE.md)** - For Claude/Anthropic tools
|
||||
- **[.github/copilot-instructions.md](.github/copilot-instructions.md)** - For GitHub Copilot
|
||||
- **[.github/copilot-instructions.md](.github/copilot-instructions.md)** - For GitHub Copilot
|
||||
- **[GEMINI.md](GEMINI.md)** - For Google Gemini tools
|
||||
- **[GPT.md](GPT.md)** - For OpenAI/ChatGPT tools
|
||||
- **[.cursor/rules/dev-standard.mdc](.cursor/rules/dev-standard.mdc)** - For Cursor editor
|
||||
|
||||
-17
@@ -125,23 +125,6 @@ dialect; each package's constraint in `pyproject.toml` documents why.
|
||||
No application-level configuration changes are required for deployments
|
||||
that don't touch SQLAlchemy directly.
|
||||
|
||||
### New metric aggregates: MEDIAN, Sample Standard Deviation, Sample Variance
|
||||
|
||||
`MEDIAN`, `STDDEV_SAMP`, and `VAR_SAMP` are now available anywhere a metric
|
||||
aggregate is chosen (every chart type, SQL Lab, MCP), not only in Pivot
|
||||
Table's controls. Support is opt-in per database engine *spec class*,
|
||||
verified against a live instance before being enabled: Postgres, MySQL
|
||||
(`STDDEV_SAMP`/`VAR_SAMP` only, no `MEDIAN`), DuckDB, and Redshift (inherits
|
||||
Postgres's support, not yet separately verified) ship enabled in this
|
||||
release. Engine specs that subclass one of those (e.g. MariaDB, Aurora
|
||||
MySQL/Postgres, TimescaleDB) inherit the same support, on the same
|
||||
not-yet-independently-verified basis. Picking one of these aggregates on a
|
||||
database that has not opted in returns a clear "not supported on this
|
||||
database" error rather than a failed query. See
|
||||
`docs/sip/median-stddev-variance-aggregates.md` for the full design
|
||||
rationale, including why this is safe to add without reintroducing the
|
||||
totals/subtotals correctness bug fixed by #41184 (SIP-216).
|
||||
|
||||
### Soft delete is on by default, and purging is live
|
||||
|
||||
`SOFT_DELETE` now ships **on** (`DEFAULT_FEATURE_FLAGS`), so deleting a
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# SIP: System-wide MEDIAN, Sample Standard Deviation, and Sample Variance metric aggregates
|
||||
|
||||
## [DRAFT — proposal for discussion]
|
||||
|
||||
This document now has an accompanying implementation in this PR, for the
|
||||
proposed mechanism plus a first, empirically-verified engine set (Postgres,
|
||||
MySQL, DuckDB, Redshift by inheritance). It is intended to seed discussion on
|
||||
whether this is the right shape and scope before it goes up for a formal SIP
|
||||
vote, not to pre-empt that discussion, the code exists so reviewers have a
|
||||
concrete design to react to rather than a description of one.
|
||||
|
||||
## Motivation
|
||||
|
||||
Before #41184 (SIP-216, the non-additive-totals fix), the Pivot Table chart
|
||||
exposed an "Aggregation function" control with 18 choices, including
|
||||
`Median`, `Sample Standard Deviation`, `Sample Variance`, `First`, `Last`,
|
||||
`Count Unique Values`, and `List Unique Values`. #41184 deleted that control
|
||||
wholesale, and deliberately so: it re-aggregated already-aggregated cell
|
||||
values to compute totals/subtotals, which is exactly the class of bug
|
||||
SIP-216 fixed (summing per-group averages, averaging per-group medians, etc.
|
||||
produces silently wrong totals). #42761 subsequently restored the one piece
|
||||
of that control's functionality that was cleanly separable from the
|
||||
correctness bug, the "show as % of row/column/total" display option,
|
||||
redesigned as a decoupled, post-hoc-only `showValuesAs` control.
|
||||
|
||||
A user has since noticed that several of the other pre-#41184 options never
|
||||
came back. Checking today's metric aggregate list (`AVG, COUNT,
|
||||
COUNT_DISTINCT, MAX, MIN, SUM`, see
|
||||
`superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`),
|
||||
most of these have a reasonable equivalent already: `Count Unique Values`
|
||||
maps to `COUNT_DISTINCT`; `Count`/`Average`/`Max`/`Min` are already standard
|
||||
aggregates; the two "fraction of" variants are already covered by
|
||||
`showValuesAs`. But `Median`, `Sample Standard Deviation`, and `Sample
|
||||
Variance` have no equivalent today anywhere in Superset, not just in Pivot
|
||||
Table, in any chart type, since the aggregate list is shared across the
|
||||
whole app.
|
||||
|
||||
This is a real, currently-live gap, not a hypothetical one:
|
||||
`superset/mcp_service/chart/chart_utils.py`, `schemas.py`, and
|
||||
`prompts/create_chart_guided.py` already treat `STDDEV`, `VAR`, `MEDIAN`,
|
||||
and `PERCENTILE` as valid aggregate values in their own validation and
|
||||
documentation, but those values are never recognized by
|
||||
`superset/connectors/sqla/models.py`'s `sqla_aggregations` dict (the actual
|
||||
mapping from aggregate name to SQL), so an AI agent using the MCP tool to
|
||||
build a chart with `"aggregate": "STDDEV"` today creates a chart that
|
||||
**errors at query time** with "Adhoc metric aggregate is invalid." This SIP
|
||||
proposes closing that gap for real, at the source, rather than patching
|
||||
around it in MCP.
|
||||
|
||||
## Proposed change
|
||||
|
||||
Add `MEDIAN`, `STDDEV_SAMP`, and `VAR_SAMP` as first-class, system-wide
|
||||
metric aggregates, available anywhere a metric aggregate is chosen (every
|
||||
chart type, SQL Lab metric picker, MCP), not as a Pivot-Table-specific
|
||||
control.
|
||||
|
||||
**Why this is safe with respect to SIP-216, and needs no Pivot-Table-specific
|
||||
code at all:** Pivot Table's non-additive-totals machinery
|
||||
(`superset-frontend/plugins/plugin-chart-pivot-table/src/plugin/utilities.ts`)
|
||||
already classifies any metric aggregate not in `ADDITIVE_AGGREGATES = {SUM,
|
||||
COUNT, MIN, MAX}` as non-additive, which routes totals/subtotals through the
|
||||
correct DB-`GROUPING SETS`-rollup path rather than client-side
|
||||
re-aggregation (`AVG` and `COUNT_DISTINCT` already go through this path
|
||||
today). `MEDIAN`/`STDDEV_SAMP`/`VAR_SAMP` fall into that bucket
|
||||
automatically, with zero changes needed to the additivity logic. So once
|
||||
these are valid, buildable SQL aggregates, Pivot Table (and every other
|
||||
chart) gets correct behavior for free. This is the version of "restore the
|
||||
control" that does not reopen the bug that was just fixed.
|
||||
|
||||
**Where the actual change needs to land, and what this PR does:**
|
||||
|
||||
1. **Done.** `superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`,
|
||||
extended the `Aggregate` type.
|
||||
2. **Done.** `superset-frontend/src/explore/constants.ts`, added to `AGGREGATES`
|
||||
(drives `AGGREGATES_OPTIONS`, the dropdown in `AdhocMetricEditPopover`).
|
||||
3. **Done**, but not consolidated. `superset/connectors/sqla/models.py`
|
||||
(`sqla_aggregations`) and `superset/models/helpers.py`
|
||||
(`ExploreMixin.sqla_aggregations`) are both wired to consult the new
|
||||
`BaseEngineSpec.get_extended_aggregation_func`, in addition to their
|
||||
existing 6-aggregate dict, so neither's original, already-tested behavior
|
||||
changed. They remain two separate dicts, consolidating them into one
|
||||
source of truth is left as a follow-up (see Open questions).
|
||||
4. **Done**, and it surfaced a second, smaller bug on top of the one this SIP
|
||||
opened with: MCP's own aggregate names (`STDDEV`, `VAR`) never matched any
|
||||
real Superset aggregate, before or after this PR, they were always going
|
||||
to error regardless of what this SIP does. `superset/mcp_service/chart/*`
|
||||
now accepts the old shorthand as an alias, normalized to the real,
|
||||
unambiguous names (`STDDEV_SAMP`, `VAR_SAMP`) this PR ships, and the guided
|
||||
prompt text points at the correct names going forward. `MEDIAN`/
|
||||
`PERCENTILE` were already spelled correctly in MCP; `PERCENTILE` remains
|
||||
unimplemented (it needs a parameter this schema has no field for) and is
|
||||
unchanged by this PR, out of scope here.
|
||||
|
||||
**The part that needs real engineering care, this must not be a blind
|
||||
`sa.func.MEDIAN` / `sa.func.STDDEV_SAMP` / `sa.func.VAR_SAMP`:**
|
||||
|
||||
`sqla_aggregations` today is a flat, engine-unaware dict (`sa.func.AVG`,
|
||||
etc., SQLAlchemy emits whatever function name it is given, with zero
|
||||
validation that the target dialect actually has it). Superset already has
|
||||
precedent for exactly this class of per-engine capability difference:
|
||||
`BaseEngineSpec.supports_grouping_sets` and `_time_grain_expressions`, both
|
||||
introduced by #41184 itself. This SIP proposes the same shape, a new
|
||||
per-engine-overridable mechanism (for example
|
||||
`BaseEngineSpec.get_aggregate_sql(aggregate, column)` with a sensible
|
||||
default, overridden per engine spec where the default does not hold),
|
||||
rather than a single hardcoded dict.
|
||||
|
||||
Verified findings so far (via `sqlglot.transpile`, cross-checked against
|
||||
known engine docs; **not** exhaustively tested against live databases, that
|
||||
is necessary follow-up work this SIP alone cannot complete):
|
||||
|
||||
| Engine | `MEDIAN(x)` | `STDDEV_SAMP(x)` | `VAR_SAMP(x)` |
|
||||
|---|---|---|---|
|
||||
| Postgres | `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY x)` | native | native |
|
||||
| MySQL | no native equivalent, needs explicit "unsupported" handling, not a blind emit | native | MySQL's `VARIANCE()` is an alias for `VAR_POP` (population), not `VAR_SAMP` (sample); a naive dialect-name substitution would silently compute the wrong statistic and needs an explicit, verified expression instead |
|
||||
| SQLite | only if the specific build was compiled with the (non-default) `SQLITE_ENABLE_PERCENTILE` extension (added in SQLite 3.43, 2023), cannot be assumed available | not available in core SQLite | not available in core SQLite |
|
||||
| BigQuery / Snowflake / DuckDB / Redshift / Oracle / T-SQL / Databricks / Spark | native `MEDIAN(x)` | native | native on BigQuery/Snowflake/Databricks/Spark, where `VARIANCE` is correctly sample variance; T-SQL has no function named `VARIANCE` at all and needs `VAR(x)` instead |
|
||||
| Trino / Presto / Hive | `PERCENTILE_CONT` / `approx_percentile` (dialect- and exactness-dependent) | native | `variance` is correctly sample variance per Trino/Presto docs |
|
||||
|
||||
This table is deliberately not exhaustive, Superset has roughly 75
|
||||
`db_engine_specs` files. The proposed default (`BaseEngineSpec`) should be
|
||||
the safe choice (mark unsupported, surface a clear user-facing error) rather
|
||||
than an optimistic one, with individual engine specs opting in once
|
||||
verified. Ship for the handful of engines above first, extend
|
||||
opportunistically.
|
||||
|
||||
**`Count Unique Values`, `First`, `Last`, `List Unique Values`, explicitly
|
||||
out of scope for this SIP:**
|
||||
|
||||
- `Count Unique Values` needs no work, it is already `COUNT_DISTINCT`.
|
||||
- `First`/`Last` have no well-defined, unambiguous meaning as a plain
|
||||
`GROUP BY` aggregate without an explicit ordering; most engines only
|
||||
support this via window functions (`FIRST_VALUE`/`LAST_VALUE` `OVER
|
||||
(ORDER BY ...)`) or do not support it as a simple aggregate at all
|
||||
(Postgres has neither built in). Restoring this properly would mean
|
||||
designing an "order by" sub-control on the metric, a real, separate
|
||||
feature, not a one-line aggregate addition. Proposed as a follow-up SIP if
|
||||
there is demand.
|
||||
- `List Unique Values` maps to the `STRING_AGG`/`GROUP_CONCAT`/`LISTAGG`/
|
||||
`ARRAY_AGG(DISTINCT ...)` family, real dialect differences, plus an open
|
||||
UX question (unbounded cell content for high-cardinality columns).
|
||||
Proposed as a follow-up SIP.
|
||||
|
||||
## New or changed public interfaces
|
||||
|
||||
- New `Aggregate` values (`MEDIAN`, `STDDEV_SAMP`, `VAR_SAMP`) selectable
|
||||
anywhere the standard metric control appears, every chart type, not just
|
||||
Pivot Table.
|
||||
- New `BaseEngineSpec` extensibility point for per-engine aggregate SQL
|
||||
generation (exact shape TBD in implementation, likely mirrors
|
||||
`_time_grain_expressions`).
|
||||
- No REST API surface changes beyond the existing metric aggregate field
|
||||
accepting new values.
|
||||
|
||||
## Migration plan and compatibility
|
||||
|
||||
No new tables/columns needed for the aggregate addition itself.
|
||||
|
||||
Restoring prior chart settings, the way #42761 restored `show_values_as` for
|
||||
charts that had it before #41184, is murkier here than it was for that PR
|
||||
and needs its own design pass: the old `aggregate_function` was a single
|
||||
Pivot-Table-level setting applied uniformly to every metric on the chart,
|
||||
not a per-metric property. A chart that had `aggregate_function: Median`
|
||||
before #41184, with a metric of `SUM(sales)`, was already silently wrong
|
||||
under the old architecture (that is the bug that was fixed); mechanically
|
||||
rewriting its metric to `MEDIAN(sales)` on upgrade would change what the
|
||||
chart's leaf cells display, not just its totals, which may not match user
|
||||
intent. This SIP proposes a best-effort, flagged-for-review migration
|
||||
(surface affected charts to admins rather than silently rewriting them)
|
||||
rather than a fully automatic one-to-one restoration.
|
||||
|
||||
## Rejected alternatives
|
||||
|
||||
- **Restoring the old `aggregateFunction` Pivot-Table control as-is.**
|
||||
Rejected: this is the literal mechanism SIP-216 removed because it
|
||||
reintroduces incorrect totals for non-additive metrics. Any fix has to go
|
||||
through the metric's own aggregate, not a separate pivot-level override.
|
||||
- **Routing all metric SQL generation through `sqlglot` expression-building
|
||||
instead of SQLAlchemy's `sa.func`.** More architecturally thorough (would
|
||||
give correct dialect syntax for free across more of the roughly 75 engine
|
||||
specs), but a much larger, more invasive change to a hot path used by
|
||||
every chart query. Noted as a possible future direction, not this SIP's
|
||||
scope; this SIP proposes the smaller, `supports_grouping_sets`-shaped
|
||||
extensibility point instead.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Resolved for this PR, worth confirming as the community's preferred
|
||||
shape:** implemented as `BaseEngineSpec._extended_aggregations` (a
|
||||
`{aggregate_name: sqla_column -> sqla_column}` dict) plus a
|
||||
`get_extended_aggregation_func` accessor, set on the concrete or shared
|
||||
base engine spec class per engine (e.g. on `PostgresBaseEngineSpec` so
|
||||
Redshift inherits it, but *not* on `PrestoBaseEngineSpec` so Hive/Spark/
|
||||
Databricks don't silently inherit unverified behavior, mirroring how
|
||||
`supports_grouping_sets` is opted into per-concrete-engine there today).
|
||||
Did not route through the `superset/sql/dialects/` sqlglot-based layer;
|
||||
that layer is for SQL Lab parsing, wiring it into chart-metric query
|
||||
building felt like a separate, larger change from this SIP's scope.
|
||||
- **Still open, not addressed in this PR:** how aggressively should
|
||||
`MEDIAN` degrade on engines without a native or exact equivalent?
|
||||
Trino/Presto/Hive were left unimplemented (unsupported) specifically to
|
||||
avoid silently answering this with an approximate function
|
||||
(`approx_percentile`) that changes the semantics of what a user asked
|
||||
for. If someone wants `MEDIAN` on those engines, this needs a real
|
||||
decision: require explicit opt-in, show a UI warning, or keep it
|
||||
disallowed.
|
||||
- **Resolved for this PR:** left the two `sqla_aggregations` dicts
|
||||
(`connectors/sqla/models.py` and `models/helpers.py`) unconsolidated,
|
||||
both now separately wired to the same new `get_extended_aggregation_func`
|
||||
hook. Consolidating them into one source of truth is real but unrelated
|
||||
cleanup, not bundled here to keep the diff reviewable.
|
||||
- **New, from implementation:** only Postgres, MySQL (partial), DuckDB, and
|
||||
Redshift (by inheritance, unverified) ship enabled. BigQuery, Snowflake,
|
||||
Trino, Presto, Hive, Spark, Databricks, Oracle, and T-SQL all have
|
||||
documented (not live-verified) support per the table above but are not
|
||||
yet wired up, each needs the same live-instance verification treatment
|
||||
before being enabled, this PR intentionally didn't guess.
|
||||
@@ -14,7 +14,7 @@
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Change Log
|
||||
|
||||
@@ -25,11 +25,8 @@ export type Aggregate =
|
||||
| 'COUNT'
|
||||
| 'COUNT_DISTINCT'
|
||||
| 'MAX'
|
||||
| 'MEDIAN'
|
||||
| 'MIN'
|
||||
| 'STDDEV_SAMP'
|
||||
| 'SUM'
|
||||
| 'VAR_SAMP';
|
||||
| 'SUM';
|
||||
|
||||
export interface AdhocMetricBase {
|
||||
hasCustomLabel?: boolean;
|
||||
|
||||
@@ -290,25 +290,6 @@ test('isAdditiveMetric: non-additive aggregates, SQL, and saved metrics are not
|
||||
expect(isAdditiveMetric('count')).toBe(false);
|
||||
});
|
||||
|
||||
test('isAdditiveMetric: MEDIAN/STDDEV_SAMP/VAR_SAMP are non-additive, with no dedicated code needed', () => {
|
||||
// Regression guard: MEDIAN/STDDEV_SAMP/VAR_SAMP are new system-wide metric
|
||||
// aggregates (not pivot-table-specific). They must fall outside
|
||||
// ADDITIVE_AGGREGATES so totals/subtotals route through the correct
|
||||
// DB-rollup path automatically, same as AVG/COUNT_DISTINCT already do --
|
||||
// averaging per-group medians (or variances) is exactly the class of bug
|
||||
// SIP-216 fixed for AVG, and would be equally wrong here.
|
||||
(['MEDIAN', 'STDDEV_SAMP', 'VAR_SAMP'] as const).forEach(aggregate => {
|
||||
expect(
|
||||
isAdditiveMetric({
|
||||
expressionType: 'SIMPLE',
|
||||
aggregate,
|
||||
column: { column_name: 'num' },
|
||||
label: `${aggregate.toLowerCase()}_num`,
|
||||
} as QueryFormMetric),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('allMetricsAdditive: all additive vs any non-additive vs empty', () => {
|
||||
const sum = {
|
||||
expressionType: 'SIMPLE',
|
||||
|
||||
+17
@@ -43,6 +43,23 @@ describe('SaveDatasetActionButton', () => {
|
||||
expect(saveDatasetBtn).toBeVisible();
|
||||
});
|
||||
|
||||
test('disables only the dataset button when canSaveDataset is false', () => {
|
||||
const onSaveAsExplore = jest.fn();
|
||||
render(
|
||||
<SaveDatasetActionButton
|
||||
setShowSave={() => true}
|
||||
onSaveAsExplore={onSaveAsExplore}
|
||||
canSaveDataset={false}
|
||||
/>,
|
||||
);
|
||||
|
||||
// Saving the query needs no results.
|
||||
expect(screen.getByRole('button', { name: 'Save' })).toBeEnabled();
|
||||
expect(
|
||||
screen.getByRole('button', { name: /save dataset/i }),
|
||||
).toBeDisabled();
|
||||
});
|
||||
|
||||
test('disables the save dataset button when the query did not run successfully', async () => {
|
||||
render(
|
||||
<SaveDatasetActionButton
|
||||
|
||||
@@ -19,12 +19,14 @@
|
||||
import { act, type ComponentProps } from 'react';
|
||||
import {
|
||||
cleanup,
|
||||
createStore,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
userEvent,
|
||||
waitFor,
|
||||
} from 'spec/helpers/testing-library';
|
||||
import reducerIndex from 'spec/helpers/reducerIndex';
|
||||
import fetchMock from 'fetch-mock';
|
||||
import { SaveDatasetModal } from 'src/SqlLab/components/SaveDatasetModal';
|
||||
import { createDatasource } from 'src/SqlLab/actions/sqlLab';
|
||||
@@ -63,6 +65,12 @@ beforeEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// In-body restores are skipped when an assertion throws, leaking a
|
||||
// configured spy into later tests.
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
// Mock createDatasource to return a thunk that resolves with the dataset's
|
||||
// new id. The test's mock store includes redux-thunk middleware (from RTK's
|
||||
// getDefaultMiddleware), so dispatch(createDatasource(...)) properly unwraps
|
||||
@@ -518,6 +526,39 @@ describe('SaveDatasetModal', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('surfaces the error and keeps the modal open when saving fails', async () => {
|
||||
// The chart-payload step's toast was built but never dispatched, so a
|
||||
// failure there was silent.
|
||||
const postFormData = jest.spyOn(
|
||||
require('src/explore/exploreUtils/formData'),
|
||||
'postFormData',
|
||||
);
|
||||
postFormData.mockRejectedValue(new Error('Boom'));
|
||||
const onHide = jest.fn();
|
||||
const store = createStore({ user }, reducerIndex);
|
||||
|
||||
render(<SaveDatasetModal {...mockedProps} onHide={onHide} />, { store });
|
||||
|
||||
fireEvent.change(screen.getByDisplayValue(/unimportant/i), {
|
||||
target: { value: 'my dataset' },
|
||||
});
|
||||
userEvent.click(screen.getByRole('button', { name: /save/i }));
|
||||
|
||||
// `createStore` builds its reducer map at runtime, so state isn't typed.
|
||||
const toasts = () =>
|
||||
(
|
||||
store.getState() as unknown as {
|
||||
messageToasts: { toastType: string }[];
|
||||
}
|
||||
).messageToasts;
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toasts()).toHaveLength(1);
|
||||
});
|
||||
expect(toasts()[0].toastType).toBe('DANGER_TOAST');
|
||||
expect(onHide).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('clearDatasetCache is imported and available', () => {
|
||||
const { clearDatasetCache } = require('src/utils/cachedSupersetGet');
|
||||
|
||||
|
||||
@@ -61,6 +61,9 @@ import type Subject from 'src/types/Subject';
|
||||
import { openInNewTab, redirect } from 'src/utils/navigationUtils';
|
||||
import { mapSubjectValuesToIds } from 'src/features/subjects/SubjectPicker';
|
||||
|
||||
// Derived so it can't drift from what `getClientErrorObject` accepts.
|
||||
type SaveErrorSource = Parameters<typeof getClientErrorObject>[0];
|
||||
|
||||
interface QueryDatabase {
|
||||
id?: number;
|
||||
}
|
||||
@@ -391,9 +394,18 @@ export const SaveDatasetModal = ({
|
||||
setDatasetName(getDefaultDatasetName());
|
||||
onHide();
|
||||
})
|
||||
.catch(() => {
|
||||
.catch((error?: SaveErrorSource) => {
|
||||
setLoading(false);
|
||||
addDangerToast(t('An error occurred saving dataset'));
|
||||
// `createDatasource` already toasted the server's message and rejects
|
||||
// with nothing; only the chart-payload step needs its own.
|
||||
if (!error) {
|
||||
return;
|
||||
}
|
||||
getClientErrorObject(error).then(e =>
|
||||
dispatch(
|
||||
addDangerToast(e.error || t('An error occurred saving dataset')),
|
||||
),
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ import {
|
||||
import SaveQuery from 'src/SqlLab/components/SaveQuery';
|
||||
import { initialState, databases } from 'src/SqlLab/fixtures';
|
||||
|
||||
const RESULT_COLUMNS = [{ column_name: 'col', type: 'STRING' }];
|
||||
|
||||
const mockedProps = {
|
||||
queryEditorId: '123',
|
||||
animation: false,
|
||||
@@ -35,7 +37,6 @@ const mockedProps = {
|
||||
onSave: () => {},
|
||||
saveQueryWarning: null,
|
||||
columns: [],
|
||||
canSaveDataset: true,
|
||||
};
|
||||
|
||||
const mockState = {
|
||||
@@ -60,8 +61,31 @@ const splitSaveBtnProps = {
|
||||
...mockedProps.database,
|
||||
allows_virtual_table_explore: true,
|
||||
},
|
||||
columns: RESULT_COLUMNS,
|
||||
};
|
||||
|
||||
const EDITOR_SQL = 'SELECT * FROM t';
|
||||
|
||||
const stateWithLatestQuery = ({
|
||||
id,
|
||||
state,
|
||||
sql = EDITOR_SQL,
|
||||
}: {
|
||||
id: string;
|
||||
state: string;
|
||||
sql?: string;
|
||||
}) => ({
|
||||
...mockState,
|
||||
sqlLab: {
|
||||
...mockState.sqlLab,
|
||||
queryEditors: mockState.sqlLab.queryEditors.map(qe => ({
|
||||
...qe,
|
||||
latestQueryId: id,
|
||||
})),
|
||||
queries: { [id]: { id, state, sql } },
|
||||
},
|
||||
});
|
||||
|
||||
const middlewares = [thunk];
|
||||
const mockStore = configureStore(middlewares);
|
||||
|
||||
@@ -97,6 +121,71 @@ describe('SavedQuery', () => {
|
||||
expect(saveBtn).toBeVisible();
|
||||
});
|
||||
|
||||
test('blocks "Save dataset" until the query has run successfully', () => {
|
||||
// Without a successful run the save can only fail server-side.
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'failed' })),
|
||||
});
|
||||
|
||||
expect(
|
||||
screen.getByRole('button', { name: /save dataset/i }),
|
||||
).toBeDisabled();
|
||||
// Saving the query itself is unaffected.
|
||||
expect(screen.getByRole('button', { name: 'Save' })).toBeEnabled();
|
||||
});
|
||||
|
||||
test('blocks "Save dataset" when no query has been run at all', () => {
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(mockState),
|
||||
});
|
||||
|
||||
expect(
|
||||
screen.getByRole('button', { name: /save dataset/i }),
|
||||
).toBeDisabled();
|
||||
});
|
||||
|
||||
test('blocks "Save dataset" when the SQL changed after a successful run', () => {
|
||||
// The run succeeded, but not for what is in the editor now -- and it is
|
||||
// the editor's SQL that gets saved.
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(
|
||||
stateWithLatestQuery({
|
||||
id: 'qid-1',
|
||||
state: 'success',
|
||||
sql: 'SELECT 1 AS ran_earlier',
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
expect(
|
||||
screen.getByRole('button', { name: /save dataset/i }),
|
||||
).toBeDisabled();
|
||||
});
|
||||
|
||||
test('blocks "Save dataset" when the successful query returned no columns', () => {
|
||||
// e.g. a DDL/DML statement -- there is nothing to introspect into a dataset.
|
||||
render(<SaveQuery {...splitSaveBtnProps} columns={[]} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
|
||||
});
|
||||
|
||||
expect(
|
||||
screen.getByRole('button', { name: /save dataset/i }),
|
||||
).toBeDisabled();
|
||||
});
|
||||
|
||||
test('enables "Save dataset" once the query has succeeded', () => {
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
|
||||
});
|
||||
|
||||
expect(screen.getByRole('button', { name: /save dataset/i })).toBeEnabled();
|
||||
});
|
||||
|
||||
test('renders a save query modal when user clicks save button', () => {
|
||||
render(<SaveQuery {...mockedProps} />, {
|
||||
useRedux: true,
|
||||
@@ -234,7 +323,7 @@ describe('SavedQuery', () => {
|
||||
test('renders a save dataset modal when user clicks "save dataset" menu item', async () => {
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(mockState),
|
||||
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
|
||||
});
|
||||
|
||||
const saveDatasetMenuItem = await screen.findByLabelText(/save dataset/i);
|
||||
@@ -248,7 +337,7 @@ describe('SavedQuery', () => {
|
||||
test('renders the save dataset modal UI', async () => {
|
||||
render(<SaveQuery {...splitSaveBtnProps} />, {
|
||||
useRedux: true,
|
||||
store: mockStore(mockState),
|
||||
store: mockStore(stateWithLatestQuery({ id: 'qid-1', state: 'success' })),
|
||||
});
|
||||
const saveDatasetMenuItem = await screen.findByLabelText(/save dataset/i);
|
||||
userEvent.click(saveDatasetMenuItem);
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
* under the License.
|
||||
*/
|
||||
import { useState, useEffect, useMemo, ChangeEvent } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
import { Query, QueryState } from '@superset-ui/core';
|
||||
import type { DatabaseObject } from 'src/features/databases/types';
|
||||
import { t } from '@apache-superset/core/translation';
|
||||
import { styled } from '@apache-superset/core/theme';
|
||||
@@ -37,7 +39,7 @@ import {
|
||||
} from 'src/SqlLab/components/SaveDatasetModal';
|
||||
import { getDatasourceAsSaveableDataset } from 'src/utils/datasourceUtils';
|
||||
import useQueryEditor from 'src/SqlLab/hooks/useQueryEditor';
|
||||
import { QueryEditor } from 'src/SqlLab/types';
|
||||
import { QueryEditor, SqlLabRootState } from 'src/SqlLab/types';
|
||||
import useLogAction from 'src/logger/useLogAction';
|
||||
import {
|
||||
LOG_ACTIONS_SQLLAB_CREATE_CHART,
|
||||
@@ -52,7 +54,6 @@ interface SaveQueryProps {
|
||||
onUpdate: (arg0: QueryPayload, id: string) => void;
|
||||
saveQueryWarning: string | null;
|
||||
database: Partial<DatabaseObject> | undefined;
|
||||
canSaveDataset: boolean;
|
||||
}
|
||||
|
||||
export type QueryPayload = {
|
||||
@@ -82,7 +83,6 @@ const SaveQuery = ({
|
||||
saveQueryWarning,
|
||||
database,
|
||||
columns,
|
||||
canSaveDataset,
|
||||
}: SaveQueryProps) => {
|
||||
const queryEditor = useQueryEditor(queryEditorId, [
|
||||
'autorun',
|
||||
@@ -113,6 +113,17 @@ const SaveQuery = ({
|
||||
const [label, setLabel] = useState<string>(defaultLabel);
|
||||
const [showSave, setShowSave] = useState<boolean>(false);
|
||||
const [showSaveDatasetModal, setShowSaveDatasetModal] = useState(false);
|
||||
// Saving a dataset runs the SQL to introspect columns, so it needs a
|
||||
// successful run of the SQL being saved that produced at least one column
|
||||
// -- editing after a run invalidates it, and running a selection only
|
||||
// validates that selection.
|
||||
const latestQuery = useSelector<SqlLabRootState, Query | undefined>(
|
||||
({ sqlLab }) => sqlLab.queries[queryEditor.latestQueryId || ''],
|
||||
);
|
||||
const canSaveDataset =
|
||||
latestQuery?.state === QueryState.Success &&
|
||||
latestQuery.sql === queryEditor.sql &&
|
||||
columns.length > 0;
|
||||
const isSaved = !!query.remoteId;
|
||||
const isLabelEmpty = label.trim().length === 0;
|
||||
const canExploreDatabase = !!database?.allows_virtual_table_explore;
|
||||
|
||||
@@ -356,7 +356,10 @@ describe('SqlEditor', () => {
|
||||
);
|
||||
|
||||
test('enables the save dataset button when the latest query succeeded', async () => {
|
||||
const { findByRole } = setupWithLatestQuery({ state: QueryState.Success });
|
||||
const { findByRole } = setupWithLatestQuery({
|
||||
state: QueryState.Success,
|
||||
sql: mockedProps.queryEditor.sql,
|
||||
});
|
||||
expect(await findByRole('button', { name: 'Save dataset' })).toBeEnabled();
|
||||
});
|
||||
|
||||
|
||||
@@ -868,7 +868,6 @@ const SqlEditor: FC<Props> = ({
|
||||
}
|
||||
saveQueryWarning={saveQueryWarning}
|
||||
database={database}
|
||||
canSaveDataset={successful && resultColumns.length > 0}
|
||||
/>
|
||||
<ShareSqlLabQuery queryEditorId={queryEditor.id} />
|
||||
</>
|
||||
|
||||
@@ -191,29 +191,6 @@ describe('AdhocMetric', () => {
|
||||
expect(adhocMetric2.inferSqlExpressionAggregate()).toBeNull();
|
||||
});
|
||||
|
||||
test('can infer the new extended aggregates (STDDEV_SAMP/VAR_SAMP/MEDIAN) from sql expressions', () => {
|
||||
const stddevSamp = new AdhocMetric({
|
||||
expressionType: EXPRESSION_TYPES.SQL,
|
||||
sqlExpression: 'STDDEV_SAMP(my_column)',
|
||||
});
|
||||
expect(stddevSamp.inferSqlExpressionColumn()).toBe('my_column');
|
||||
expect(stddevSamp.inferSqlExpressionAggregate()).toBe('STDDEV_SAMP');
|
||||
|
||||
const varSamp = new AdhocMetric({
|
||||
expressionType: EXPRESSION_TYPES.SQL,
|
||||
sqlExpression: 'VAR_SAMP(my_column)',
|
||||
});
|
||||
expect(varSamp.inferSqlExpressionColumn()).toBe('my_column');
|
||||
expect(varSamp.inferSqlExpressionAggregate()).toBe('VAR_SAMP');
|
||||
|
||||
const median = new AdhocMetric({
|
||||
expressionType: EXPRESSION_TYPES.SQL,
|
||||
sqlExpression: 'MEDIAN(my_column)',
|
||||
});
|
||||
expect(median.inferSqlExpressionColumn()).toBe('my_column');
|
||||
expect(median.inferSqlExpressionAggregate()).toBe('MEDIAN');
|
||||
});
|
||||
|
||||
test('will infer columns and aggregates when converting to a simple expression', () => {
|
||||
const adhocMetric = new AdhocMetric({
|
||||
expressionType: EXPRESSION_TYPES.SQL,
|
||||
@@ -271,20 +248,4 @@ describe('AdhocMetric', () => {
|
||||
).toBe('COUNT_DISTINCT');
|
||||
expect(emptyColumnName.getDefaultLabel()).toBe('COUNT_DISTINCT');
|
||||
});
|
||||
|
||||
test('should prefill a portable MEDIAN expression for the Custom SQL tab, but keep the raw label', () => {
|
||||
const median = new AdhocMetric({
|
||||
column: valueColumn,
|
||||
aggregate: AGGREGATES.MEDIAN,
|
||||
hasCustomLabel: false,
|
||||
});
|
||||
// MEDIAN(column) isn't valid SQL on every engine this PR verifies it
|
||||
// for (e.g. PostgreSQL has no MEDIAN function), so the editable Custom
|
||||
// SQL tab is prefilled with the portable, standards-based spelling.
|
||||
expect(median.translateToSql({ transformCountDistinct: true })).toBe(
|
||||
'PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value)',
|
||||
);
|
||||
// The display label stays the concise, human-readable form.
|
||||
expect(median.getDefaultLabel()).toBe('MEDIAN(value)');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -153,19 +153,6 @@ export default class AdhocMetric {
|
||||
) {
|
||||
return `COUNT(DISTINCT ${column.slice(1, -1)})`;
|
||||
}
|
||||
// MEDIAN(column) isn't a real function on every engine this PR
|
||||
// verifies it for -- PostgreSQL/Redshift compile it to
|
||||
// PERCENTILE_CONT(0.5) WITHIN GROUP instead. `transformCountDistinct`
|
||||
// signals this call is prefilling the *editable, executable* Custom
|
||||
// SQL tab (not just a display label), so use the portable,
|
||||
// standards-based spelling there instead of the raw aggregate name.
|
||||
if (
|
||||
params.transformCountDistinct &&
|
||||
aggregate === AGGREGATES.MEDIAN &&
|
||||
/^\(.*\)$/.test(column)
|
||||
) {
|
||||
return `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${column.slice(1, -1)})`;
|
||||
}
|
||||
return aggregate + column;
|
||||
}
|
||||
if (this.expressionType === EXPRESSION_TYPES.SQL) {
|
||||
|
||||
+1
-2
@@ -36,7 +36,6 @@ import {
|
||||
import sqlKeywords from 'src/SqlLab/utils/sqlKeywords';
|
||||
import { noOp } from 'src/utils/common';
|
||||
import {
|
||||
AGGREGATES_LABELS,
|
||||
AGGREGATES_OPTIONS,
|
||||
POPOVER_INITIAL_HEIGHT,
|
||||
POPOVER_INITIAL_WIDTH,
|
||||
@@ -549,7 +548,7 @@ function AdhocMetricEditPopover({
|
||||
<Select
|
||||
options={AGGREGATES_OPTIONS.map(option => ({
|
||||
value: option,
|
||||
label: AGGREGATES_LABELS[option] ?? option,
|
||||
label: option,
|
||||
key: option,
|
||||
}))}
|
||||
{...aggregateSelectProps}
|
||||
|
||||
@@ -23,22 +23,11 @@ export const AGGREGATES = {
|
||||
COUNT: 'COUNT',
|
||||
COUNT_DISTINCT: 'COUNT_DISTINCT',
|
||||
MAX: 'MAX',
|
||||
MEDIAN: 'MEDIAN',
|
||||
MIN: 'MIN',
|
||||
STDDEV_SAMP: 'STDDEV_SAMP',
|
||||
SUM: 'SUM',
|
||||
VAR_SAMP: 'VAR_SAMP',
|
||||
};
|
||||
export const AGGREGATES_OPTIONS = Object.values(AGGREGATES);
|
||||
|
||||
// Human-readable labels for aggregates whose raw enum value isn't
|
||||
// self-explanatory in the UI. Aggregates absent here (AVG, COUNT, MAX,
|
||||
// MEDIAN, MIN, SUM, ...) are already clear as their raw value.
|
||||
export const AGGREGATES_LABELS: Record<string, string> = {
|
||||
STDDEV_SAMP: t('Sample Standard Deviation'),
|
||||
VAR_SAMP: t('Sample Variance'),
|
||||
};
|
||||
|
||||
export enum Operators {
|
||||
Equals = 'EQUALS',
|
||||
NotEquals = 'NOT_EQUALS',
|
||||
@@ -187,7 +176,7 @@ export const DISABLE_INPUT_OPERATORS = [
|
||||
export const sqlaAutoGeneratedMetricNameRegex =
|
||||
/^(sum|min|max|avg|count|count_distinct)__.*$/i;
|
||||
export const sqlaAutoGeneratedMetricRegex =
|
||||
/^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT|MEDIAN|STDDEV_SAMP|VAR_SAMP)\([A-Z0-9_."]*\)$/i;
|
||||
/^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
|
||||
|
||||
export const TIME_FILTER_LABELS = {
|
||||
time_range: t('Time range'),
|
||||
|
||||
@@ -413,12 +413,6 @@ class ChartDataRestApi(ChartRestApi):
|
||||
# for async queries with jinja context
|
||||
set_form_data(cached_data)
|
||||
query_context = self._create_query_context_from_form(cached_data)
|
||||
# Mark as a cache replay so _sql_filters_modified skips the
|
||||
# SQL-extras check. The original request already passed the
|
||||
# full security check, cache keys are opaque SHA-256 hashes
|
||||
# (unguessable), and force_cached only serves pre-computed
|
||||
# data — no new SQL is executed.
|
||||
query_context._from_cache_replay = True
|
||||
command = ChartDataCommand(query_context)
|
||||
command.validate()
|
||||
except ChartDataCacheLoadError:
|
||||
|
||||
@@ -42,7 +42,6 @@ from superset.utils import pandas_postprocessing, schema as utils
|
||||
from superset.utils.core import (
|
||||
AnnotationType,
|
||||
DatasourceType,
|
||||
EXTENDED_METRIC_AGGREGATES,
|
||||
FilterOperator,
|
||||
PostProcessingBoxplotWhiskerType,
|
||||
PostProcessingContributionOrientation,
|
||||
@@ -435,15 +434,7 @@ class ChartDataAdhocMetricSchema(Schema):
|
||||
"Only required for simple expression types."
|
||||
},
|
||||
validate=validate.OneOf(
|
||||
choices=(
|
||||
"AVG",
|
||||
"COUNT",
|
||||
"COUNT_DISTINCT",
|
||||
"MAX",
|
||||
"MIN",
|
||||
"SUM",
|
||||
*sorted(EXTENDED_METRIC_AGGREGATES),
|
||||
)
|
||||
choices=("AVG", "COUNT", "COUNT_DISTINCT", "MAX", "MIN", "SUM")
|
||||
),
|
||||
)
|
||||
column = fields.Nested(ChartDataColumnSchema)
|
||||
|
||||
@@ -33,7 +33,11 @@ from superset.commands.dataset.exceptions import (
|
||||
)
|
||||
from superset.commands.utils import populate_subjects
|
||||
from superset.daos.dataset import DatasetDAO
|
||||
from superset.exceptions import SupersetParseError, SupersetSecurityException
|
||||
from superset.exceptions import (
|
||||
SupersetException,
|
||||
SupersetParseError,
|
||||
SupersetSecurityException,
|
||||
)
|
||||
from superset.extensions import security_manager
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.decorators import on_error, transaction
|
||||
@@ -50,7 +54,25 @@ class CreateDatasetCommand(CreateMixin, BaseCommand):
|
||||
self.validate()
|
||||
|
||||
dataset = DatasetDAO.create(attributes=self._properties)
|
||||
dataset.fetch_metadata()
|
||||
try:
|
||||
dataset.fetch_metadata()
|
||||
except SupersetException as ex:
|
||||
# Not a SQLAlchemyError, so ``on_error`` re-raises it untouched and
|
||||
# it escapes to FAB's ``@safe`` as an opaque 500 "Fatal error".
|
||||
# Deliberately covers the 403 ``SupersetSecurityException`` raised
|
||||
# for mutation/multi-statement SQL too: ``validate()`` already
|
||||
# reports that class of rejection as a 422 on ``sql`` via
|
||||
# ``DatasetDataAccessIsNotAllowed``.
|
||||
raise DatasetInvalidError(
|
||||
exceptions=[
|
||||
ValidationError(
|
||||
# ``lazy_gettext`` messages aren't ``str``, so
|
||||
# marshmallow won't wrap them into a list on its own.
|
||||
[str(ex.message)],
|
||||
field_name="sql" if self._properties.get("sql") else "table",
|
||||
)
|
||||
]
|
||||
) from ex
|
||||
return dataset
|
||||
|
||||
def validate(self) -> None: # noqa: C901
|
||||
|
||||
@@ -1898,6 +1898,11 @@ class SqlaTable(
|
||||
|
||||
if expression_type == utils.AdhocMetricExpressionType.SIMPLE:
|
||||
aggregate: Any = metric.get("aggregate")
|
||||
if (
|
||||
not isinstance(aggregate, str)
|
||||
or aggregate not in self.sqla_aggregations
|
||||
):
|
||||
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
|
||||
metric_column = metric.get("column") or {}
|
||||
column_name = cast(str, metric_column.get("column_name"))
|
||||
table_column: TableColumn | None = columns_by_name.get(column_name)
|
||||
@@ -1907,27 +1912,7 @@ class SqlaTable(
|
||||
)
|
||||
else:
|
||||
sqla_column = column(column_name)
|
||||
|
||||
if isinstance(aggregate, str) and aggregate in self.sqla_aggregations:
|
||||
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
|
||||
elif isinstance(aggregate, str) and (
|
||||
extended_func := self.db_engine_spec.get_extended_aggregation_func(
|
||||
aggregate
|
||||
)
|
||||
):
|
||||
sqla_metric = extended_func(sqla_column)
|
||||
elif (
|
||||
isinstance(aggregate, str)
|
||||
and aggregate in utils.EXTENDED_METRIC_AGGREGATES
|
||||
):
|
||||
raise QueryObjectValidationError(
|
||||
_(
|
||||
"The %(aggregate)s aggregate is not supported on this database",
|
||||
aggregate=aggregate,
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
|
||||
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
|
||||
elif expression_type == utils.AdhocMetricExpressionType.SQL:
|
||||
expression: str | None = metric.get("sqlExpression")
|
||||
if not isinstance(expression, str) or not expression.strip():
|
||||
|
||||
@@ -632,33 +632,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
# issuing one query per level. Conservative default of False; engines opt in.
|
||||
supports_grouping_sets = False
|
||||
|
||||
# SQL-generating callables for metric aggregates that have no safe, universal
|
||||
# cross-dialect spelling -- unlike SUM/COUNT/AVG/MIN/MAX/COUNT_DISTINCT (see
|
||||
# `SqlaTable.sqla_aggregations`), which SQLAlchemy's generic `sa.func` can emit
|
||||
# unchanged on every engine. Keyed by `Aggregate` name (see
|
||||
# `superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts`);
|
||||
# each value takes a SQLAlchemy column and returns the aggregate expression.
|
||||
# Absent by default: an aggregate not present here is unsupported on this
|
||||
# engine, and callers must surface a clear "not supported" error rather than
|
||||
# emit unverified SQL (a wrong statistic returned silently is worse than an
|
||||
# error). Engines opt in via `get_extended_aggregation_func` below once the
|
||||
# expression has been verified against real engine behavior, not assumed
|
||||
# from syntax alone -- see the MySQL engine spec for a concrete example of
|
||||
# why this distinction matters (its `VARIANCE()` computes the *population*
|
||||
# variance, not the *sample* variance `VAR_SAMP` denotes).
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
@classmethod
|
||||
def get_extended_aggregation_func(
|
||||
cls, aggregate: str
|
||||
) -> Callable[[ColumnElement], ColumnElement] | None:
|
||||
"""
|
||||
SQL-generating callable for an aggregate not handled by the generic
|
||||
`sa.func` mapping (e.g. MEDIAN, STDDEV_SAMP, VAR_SAMP). Returns None if
|
||||
this engine has no verified, correct expression for it.
|
||||
"""
|
||||
return cls._extended_aggregations.get(aggregate)
|
||||
|
||||
# Is the DB engine spec able to change the default schema? This requires implementing # noqa: E501
|
||||
# a custom `adjust_engine_params` method.
|
||||
supports_dynamic_schema = False
|
||||
|
||||
@@ -15,10 +15,9 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
@@ -28,12 +27,6 @@ class CockroachDbEngineSpec(PostgresEngineSpec):
|
||||
engine = "cockroachdb"
|
||||
engine_name = "CockroachDB"
|
||||
|
||||
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not CockroachDB's distributed
|
||||
# query engine; disable it here until someone confirms the same expressions
|
||||
# against a live CockroachDB instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"CockroachDB is a distributed SQL database built for cloud applications."
|
||||
|
||||
@@ -17,14 +17,13 @@
|
||||
import logging
|
||||
import re
|
||||
from re import Pattern
|
||||
from typing import Any, Callable, Optional
|
||||
from typing import Any, Optional
|
||||
from urllib import parse
|
||||
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy import Float, Integer, Numeric, String, TEXT, text, types
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.type_api import TypeEngine
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
@@ -122,11 +121,6 @@ class DorisEngineSpec(MySQLEngineSpec):
|
||||
# while technically supported by Doris, this generates invalid table identifiers
|
||||
supports_cross_catalog_queries = False
|
||||
|
||||
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
|
||||
# against real MySQL behavior, not Doris's OLAP query engine; disable it here
|
||||
# until someone confirms the same expressions against a live Doris instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"Apache Doris is a high-performance real-time analytical database."
|
||||
|
||||
@@ -20,9 +20,8 @@ from __future__ import annotations
|
||||
import re
|
||||
from datetime import datetime
|
||||
from re import Pattern
|
||||
from typing import Any, Callable, TYPE_CHECKING, TypedDict
|
||||
from typing import Any, TYPE_CHECKING, TypedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from apispec import APISpec
|
||||
from apispec.ext.marshmallow import MarshmallowPlugin
|
||||
from flask import current_app as app
|
||||
@@ -31,7 +30,6 @@ from marshmallow import fields, Schema
|
||||
from sqlalchemy import text, types
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.databases.utils import make_url_safe
|
||||
@@ -204,18 +202,6 @@ class DuckDBEngineSpec(DuckDBParametersMixin, BaseEngineSpec):
|
||||
sqlalchemy_uri_placeholder = "duckdb:////path/to/duck.db"
|
||||
supports_multivalues_insert = True
|
||||
|
||||
# Verified against a live duckdb instance (in-process, no server needed),
|
||||
# including under GROUPING SETS: the grand total correctly reflects every
|
||||
# row, not an aggregate-of-aggregates. STDDEV_SAMP/VAR_SAMP values match
|
||||
# postgres/mysql exactly for the same inputs; MEDIAN matches postgres
|
||||
# (mysql has no native MEDIAN to compare against). Inherited by
|
||||
# MotherDuckEngineSpec.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
|
||||
"MEDIAN": sa.func.median,
|
||||
"STDDEV_SAMP": sa.func.stddev_samp,
|
||||
"VAR_SAMP": sa.func.var_samp,
|
||||
}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"DuckDB is an in-process OLAP database designed for fast "
|
||||
|
||||
@@ -15,10 +15,6 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
@@ -34,12 +30,6 @@ class GreenplumEngineSpec(PostgresEngineSpec):
|
||||
engine_name = "Greenplum"
|
||||
default_driver = "psycopg2"
|
||||
|
||||
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is
|
||||
# verified against real Postgres behavior, not Greenplum's MPP query engine;
|
||||
# disable it here until someone confirms the same expressions against a live
|
||||
# Greenplum instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"VMware Greenplum is a massively parallel processing (MPP) "
|
||||
|
||||
@@ -15,10 +15,9 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
@@ -30,11 +29,6 @@ class HanaEngineSpec(PostgresBaseEngineSpec):
|
||||
engine = "hana"
|
||||
engine_name = "SAP HANA"
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not HANA's; disable it here
|
||||
# until someone confirms the same expressions against a live HANA instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"SAP HANA is an in-memory relational database and application platform."
|
||||
|
||||
@@ -17,10 +17,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
|
||||
@@ -36,12 +32,6 @@ class HologresEngineSpec(PostgresBaseEngineSpec):
|
||||
engine_name = "Hologres"
|
||||
default_driver = "psycopg2"
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not Hologres's real-time analytics
|
||||
# engine; disable it here until someone confirms the same expressions against a
|
||||
# live Hologres instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"Alibaba Cloud Hologres is a real-time interactive analytics service, "
|
||||
|
||||
@@ -25,7 +25,6 @@ from re import Pattern
|
||||
from typing import Any, Callable, Optional, TYPE_CHECKING
|
||||
from urllib import parse
|
||||
|
||||
import sqlalchemy as sa
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.dialects.mysql import (
|
||||
@@ -41,7 +40,6 @@ from sqlalchemy.dialects.mysql import (
|
||||
TINYTEXT,
|
||||
)
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.db_engine_specs.base import (
|
||||
@@ -96,22 +94,6 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
|
||||
supports_dynamic_schema = True
|
||||
supports_multivalues_insert = True
|
||||
|
||||
# Verified against a live mysql:8.0 instance, including under GROUP BY ...
|
||||
# WITH ROLLUP. `STDDEV_SAMP`/`VAR_SAMP` are native, correct sample
|
||||
# statistics. MEDIAN is deliberately absent: MySQL has neither a `MEDIAN`
|
||||
# function nor `PERCENTILE_CONT` (confirmed: both error). Its `VARIANCE()`
|
||||
# function is population variance, not sample variance, so it is not a
|
||||
# valid stand-in for VAR_SAMP either.
|
||||
# Inherited by MariaDB (a MySQL fork implementing the same aggregate
|
||||
# functions) and by Aurora MySQL / its Data API variant (AWS's wire- and
|
||||
# SQL-compatible managed MySQL) -- unlike CockroachDB/Greenplum/HANA
|
||||
# relative to Postgres, none of these run a materially different query
|
||||
# engine, so no separate reset is needed.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
|
||||
"STDDEV_SAMP": sa.func.stddev_samp,
|
||||
"VAR_SAMP": sa.func.var_samp,
|
||||
}
|
||||
|
||||
metadata = {
|
||||
"description": "MySQL is a popular open-source relational database.",
|
||||
"logo": "mysql.png",
|
||||
|
||||
@@ -14,10 +14,6 @@
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
@@ -28,11 +24,6 @@ class NetezzaEngineSpec(PostgresBaseEngineSpec):
|
||||
default_driver = "nzpy"
|
||||
engine_name = "IBM Netezza Performance Server"
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not Netezza's; disable it here
|
||||
# until someone confirms the same expressions against a live Netezza instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": "IBM Netezza Performance Server is a data warehouse appliance.",
|
||||
"logo": "netezza.png",
|
||||
|
||||
@@ -17,11 +17,10 @@
|
||||
import logging
|
||||
import re
|
||||
from re import Pattern
|
||||
from typing import Any, Callable, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy import Numeric, TEXT, types
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.type_api import TypeEngine
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
@@ -85,12 +84,6 @@ class OceanBaseEngineSpec(MySQLEngineSpec):
|
||||
encryption_parameters = {"ssl": "0"}
|
||||
supports_dynamic_schema = True
|
||||
|
||||
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
|
||||
# against real MySQL behavior, not OceanBase's distributed query engine;
|
||||
# disable it here until someone confirms the same expressions against a live
|
||||
# OceanBase instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": "OceanBase is a distributed relational database.",
|
||||
"logo": "oceanbase.svg",
|
||||
|
||||
@@ -23,14 +23,12 @@ from datetime import datetime
|
||||
from re import Pattern
|
||||
from typing import Any, Callable, Optional, TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy import text, types
|
||||
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, ENUM, INTERVAL, JSON
|
||||
from sqlalchemy.dialects.postgresql.base import PGInspector
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.expression import ColumnClause
|
||||
from sqlalchemy.types import Date, DateTime, String
|
||||
|
||||
@@ -194,25 +192,6 @@ class PostgresBaseEngineSpec(BaseEngineSpec):
|
||||
TimeGrain.YEAR: "DATE_TRUNC('year', {col})",
|
||||
}
|
||||
|
||||
# Verified against a live postgres:16 instance, including under GROUPING
|
||||
# SETS (the pivot table's non-additive-total rollup pattern): the grand
|
||||
# total correctly reflects every row, not an aggregate-of-aggregates.
|
||||
# STDDEV_SAMP/VAR_SAMP (not MEDIAN -- see its override) are inherited by
|
||||
# Redshift (a Postgres fork); its SQL function reference documents the
|
||||
# same support, but that has not been separately verified against a live
|
||||
# Redshift instance.
|
||||
# Also inherited by TimescaleDB (a Postgres extension, not a forked query
|
||||
# engine -- it runs unmodified Postgres aggregate execution) and by
|
||||
# Aurora PostgreSQL / its Data API variant (AWS's wire- and
|
||||
# SQL-compatible managed Postgres). Engines that share the SQL dialect
|
||||
# but run a materially different query engine (CockroachDB, Greenplum,
|
||||
# SAP HANA) reset this to `{}` instead -- see those engine specs.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
|
||||
"MEDIAN": lambda col: sa.func.percentile_cont(0.5).within_group(col),
|
||||
"STDDEV_SAMP": sa.func.stddev_samp,
|
||||
"VAR_SAMP": sa.func.var_samp,
|
||||
}
|
||||
|
||||
custom_errors: dict[Pattern[str], tuple[str, SupersetErrorType, dict[str, Any]]] = {
|
||||
CONNECTION_INVALID_USERNAME_REGEX: (
|
||||
__('The username "%(username)s" does not exist.'),
|
||||
|
||||
@@ -20,12 +20,10 @@ import logging
|
||||
import re
|
||||
import warnings
|
||||
from re import Pattern
|
||||
from typing import Any, Callable
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import sqlalchemy as sa
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.types import NVARCHAR
|
||||
|
||||
from superset.db_engine_specs.base import BasicParametersMixin, DatabaseCategory
|
||||
@@ -279,24 +277,6 @@ class RedshiftEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec):
|
||||
"$.aws_iam.role_arn": "AWS IAM Role ARN",
|
||||
}
|
||||
|
||||
# Redshift inherits `PostgresBaseEngineSpec._extended_aggregations` for
|
||||
# STDDEV_SAMP/VAR_SAMP (plain, non-sort-based aggregate calls), but
|
||||
# overrides MEDIAN here instead of inheriting Postgres's spelling.
|
||||
# Postgres has no native MEDIAN function and compiles it to
|
||||
# `percentile_cont(0.5) WITHIN GROUP (ORDER BY col)`; Redshift, unlike
|
||||
# Postgres, documents a native `MEDIAN(x)` aggregate function. Using
|
||||
# that native spelling -- rather than the inherited WITHIN-GROUP form --
|
||||
# also sidesteps a documented Redshift restriction rejecting more than
|
||||
# one sort-based aggregate (MEDIAN, PERCENTILE_CONT, LISTAGG WITHIN
|
||||
# GROUP, ...) with a different ORDER BY in the same query, e.g.
|
||||
# `MEDIAN(sales)` alongside `MEDIAN(margin)`: a plain function call has
|
||||
# no explicit ORDER BY clause to conflict.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {
|
||||
"MEDIAN": sa.func.median,
|
||||
"STDDEV_SAMP": PostgresBaseEngineSpec._extended_aggregations["STDDEV_SAMP"],
|
||||
"VAR_SAMP": PostgresBaseEngineSpec._extended_aggregations["VAR_SAMP"],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update_params_from_encrypted_extra(
|
||||
database: Database,
|
||||
|
||||
@@ -14,10 +14,6 @@
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
@@ -30,12 +26,6 @@ class RisingWaveDbEngineSpec(PostgresEngineSpec):
|
||||
"risingwave://user:password@host:port/dbname[?key=value&key=value...]"
|
||||
)
|
||||
|
||||
# `PostgresEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is
|
||||
# verified against real Postgres behavior, not RisingWave's streaming query
|
||||
# engine; disable it here until someone confirms the same expressions against
|
||||
# a live RisingWave instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": "RisingWave is a distributed streaming database.",
|
||||
"logo": "risingwave.svg",
|
||||
|
||||
@@ -20,7 +20,7 @@ import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from re import Pattern
|
||||
from typing import Any, Callable, Optional, TYPE_CHECKING, TypedDict
|
||||
from typing import Any, Optional, TYPE_CHECKING, TypedDict
|
||||
from urllib import parse
|
||||
|
||||
from apispec import APISpec
|
||||
@@ -33,7 +33,6 @@ from marshmallow import fields, Schema
|
||||
from sqlalchemy import text, types
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.databases.utils import make_url_safe
|
||||
@@ -88,11 +87,6 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
|
||||
force_column_alias_quotes = True
|
||||
max_column_name_length = 256
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not Snowflake's; disable it here
|
||||
# until someone confirms the same expressions against a live Snowflake instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
# Snowflake doesn't support IS true/false syntax, use = true/false instead
|
||||
use_equality_for_boolean_filters = True
|
||||
|
||||
|
||||
@@ -18,14 +18,13 @@
|
||||
import logging
|
||||
import re
|
||||
from re import Pattern
|
||||
from typing import Any, Callable
|
||||
from typing import Any
|
||||
from urllib import parse
|
||||
|
||||
from flask_babel import gettext as __
|
||||
from sqlalchemy import Float, Integer, Numeric, text, types
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.type_api import TypeEngine
|
||||
|
||||
from superset import is_feature_enabled
|
||||
@@ -104,12 +103,6 @@ class StarRocksEngineSpec(MySQLEngineSpec):
|
||||
supports_dynamic_schema = True
|
||||
supports_catalog = supports_dynamic_catalog = supports_cross_catalog_queries = True
|
||||
|
||||
# `MySQLEngineSpec._extended_aggregations` (STDDEV_SAMP/VAR_SAMP) is verified
|
||||
# against real MySQL behavior, not StarRocks's OLAP query engine; disable it
|
||||
# here until someone confirms the same expressions against a live StarRocks
|
||||
# instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"StarRocks is a high-performance analytical database "
|
||||
|
||||
@@ -14,10 +14,6 @@
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
|
||||
@@ -26,11 +22,6 @@ class VerticaEngineSpec(PostgresBaseEngineSpec):
|
||||
engine = "vertica"
|
||||
engine_name = "Vertica"
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not Vertica's; disable it here
|
||||
# until someone confirms the same expressions against a live Vertica instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": "Vertica is a column-oriented analytics database.",
|
||||
"logo": "vertica.png",
|
||||
|
||||
@@ -17,10 +17,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from superset.db_engine_specs.base import DatabaseCategory
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
|
||||
@@ -36,12 +32,6 @@ class YugabyteDBEngineSpec(PostgresBaseEngineSpec):
|
||||
engine_name = "YugabyteDB"
|
||||
default_driver = "psycopg2"
|
||||
|
||||
# `PostgresBaseEngineSpec._extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP)
|
||||
# is verified against real Postgres behavior, not YugabyteDB's distributed
|
||||
# query engine; disable it here until someone confirms the same expressions
|
||||
# against a live YugabyteDB instance.
|
||||
_extended_aggregations: dict[str, Callable[[ColumnElement], ColumnElement]] = {}
|
||||
|
||||
metadata = {
|
||||
"description": (
|
||||
"YugabyteDB is a distributed SQL database built on top of PostgreSQL."
|
||||
|
||||
@@ -607,19 +607,12 @@ def create_metric_object(col: ColumnRef) -> Dict[str, Any] | str:
|
||||
"MIN",
|
||||
"MAX",
|
||||
"COUNT_DISTINCT",
|
||||
"STDDEV_SAMP",
|
||||
"VAR_SAMP",
|
||||
"STDDEV",
|
||||
"VAR",
|
||||
"MEDIAN",
|
||||
"PERCENTILE",
|
||||
}
|
||||
# Accept the pre-SIP shorthand names too, mapped onto the real,
|
||||
# unambiguous aggregate names Superset actually supports (bare
|
||||
# "STDDEV"/"VAR" are ambiguous between sample and population statistics,
|
||||
# and differ by engine -- see docs/sip/median-stddev-variance-aggregates.md).
|
||||
aggregate_aliases = {"STDDEV": "STDDEV_SAMP", "VAR": "VAR_SAMP"}
|
||||
aggregate = aggregate_aliases.get(
|
||||
(col.aggregate or "SUM").upper(), col.aggregate or "SUM"
|
||||
)
|
||||
aggregate = col.aggregate or "SUM"
|
||||
|
||||
# Validate aggregate function (final safety check)
|
||||
if aggregate.upper() not in valid_aggregates:
|
||||
|
||||
@@ -137,9 +137,7 @@ Example table config:
|
||||
- If the chart type doesn't suit the data, try a different kind
|
||||
|
||||
## Available Aggregations
|
||||
SUM, COUNT, AVG, MIN, MAX, COUNT_DISTINCT, STDDEV_SAMP, VAR_SAMP, MEDIAN
|
||||
(support for STDDEV_SAMP/VAR_SAMP/MEDIAN depends on the connected database;
|
||||
an unsupported choice returns a clear error naming the unsupported aggregate)
|
||||
SUM, COUNT, AVG, MIN, MAX, COUNT_DISTINCT, STDDEV, VAR, MEDIAN
|
||||
|
||||
## Custom SQL Metrics
|
||||
For ratio metrics, weighted averages, and conditional aggregates,
|
||||
|
||||
@@ -718,15 +718,10 @@ class ColumnRef(UnknownFieldCheckMixin):
|
||||
"MIN",
|
||||
"MAX",
|
||||
"COUNT_DISTINCT",
|
||||
"STDDEV_SAMP",
|
||||
"VAR_SAMP",
|
||||
"MEDIAN",
|
||||
"PERCENTILE",
|
||||
# Pre-SIP shorthand, accepted and normalized to the names above by
|
||||
# `chart_utils.create_metric_object`; kept here so schema
|
||||
# validation doesn't reject them before that normalization runs.
|
||||
"STDDEV",
|
||||
"VAR",
|
||||
"MEDIAN",
|
||||
"PERCENTILE",
|
||||
]
|
||||
| None
|
||||
) = Field(None, description="SQL aggregate function")
|
||||
|
||||
@@ -693,15 +693,7 @@ class DatasetValidator:
|
||||
# and text in most SQL engines, so restricting them here would
|
||||
# produce false-positive errors. Leave those to the Tier-2
|
||||
# compile check.
|
||||
numeric_aggs = [
|
||||
"SUM",
|
||||
"AVG",
|
||||
"STDDEV_SAMP",
|
||||
"VAR_SAMP",
|
||||
"MEDIAN",
|
||||
"STDDEV",
|
||||
"VAR",
|
||||
]
|
||||
numeric_aggs = ["SUM", "AVG", "STDDEV", "VAR", "MEDIAN"]
|
||||
if (
|
||||
col_ref.aggregate in numeric_aggs
|
||||
and not col_info.get("is_numeric", False)
|
||||
|
||||
@@ -217,14 +217,7 @@ class FormatTypeValidator:
|
||||
"""Suggest appropriate format based on column and aggregation."""
|
||||
if column.aggregate in ["COUNT", "COUNT_DISTINCT"]:
|
||||
return ",d" # Integer with thousands separator
|
||||
elif column.aggregate in [
|
||||
"AVG",
|
||||
"MEDIAN",
|
||||
"STDDEV_SAMP",
|
||||
"VAR_SAMP",
|
||||
"STDDEV",
|
||||
"VAR",
|
||||
]:
|
||||
elif column.aggregate in ["AVG", "STDDEV", "VAR"]:
|
||||
return ",.2f" # Two decimals for statistical measures
|
||||
elif column.aggregate in ["SUM", "MIN", "MAX"]:
|
||||
# Could be currency or regular number, default to flexible
|
||||
|
||||
@@ -3518,30 +3518,15 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
|
||||
|
||||
if expression_type == utils.AdhocMetricExpressionType.SIMPLE:
|
||||
aggregate: Any = metric.get("aggregate")
|
||||
if (
|
||||
not isinstance(aggregate, str)
|
||||
or aggregate not in self.sqla_aggregations
|
||||
):
|
||||
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
|
||||
metric_column = metric.get("column") or {}
|
||||
column_name = cast(str, metric_column.get("column_name"))
|
||||
sqla_column = sa.column(column_name)
|
||||
|
||||
if isinstance(aggregate, str) and aggregate in self.sqla_aggregations:
|
||||
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
|
||||
elif isinstance(aggregate, str) and (
|
||||
extended_func := self.db_engine_spec.get_extended_aggregation_func(
|
||||
aggregate
|
||||
)
|
||||
):
|
||||
sqla_metric = extended_func(sqla_column)
|
||||
elif (
|
||||
isinstance(aggregate, str)
|
||||
and aggregate in utils.EXTENDED_METRIC_AGGREGATES
|
||||
):
|
||||
raise QueryObjectValidationError(
|
||||
_(
|
||||
"The %(aggregate)s aggregate is not supported on this database",
|
||||
aggregate=aggregate,
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
|
||||
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
|
||||
elif expression_type == utils.AdhocMetricExpressionType.SQL:
|
||||
expression: Any = metric.get("sqlExpression")
|
||||
if not isinstance(expression, str) or not expression.strip():
|
||||
|
||||
@@ -1177,276 +1177,6 @@ def _orderby_modified(
|
||||
return False
|
||||
|
||||
|
||||
# The frontend emits ``{expressionType: "SQL", sqlExpression: "1 = 0"}`` when
|
||||
# a native Select filter has "Filter value is required" enabled and no value
|
||||
# has been selected yet (superset-frontend/src/filters/utils.ts). After
|
||||
# ``_sanitize_clause`` wraps it in parentheses the resulting ``extras.where``
|
||||
# clause is ``(1 = 0)``. This is safe — it returns zero rows — and must be
|
||||
# allowed so that embedded charts are not rejected before the user picks a
|
||||
# filter value.
|
||||
_EMPTY_FILTER_SENTINEL = "1 = 0"
|
||||
|
||||
|
||||
def _split_extras_clauses(composed: str) -> list[str]:
|
||||
"""
|
||||
Extract raw SQL expressions from a composed ``extras.where`` /
|
||||
``extras.having`` string.
|
||||
|
||||
``_sanitize_clause`` (``form_data_query_context.py:92``) /
|
||||
``processFilters.ts`` (``superset-ui-core/src/query/processFilters.ts``)
|
||||
wraps each expression in one layer of parentheses and joins them with
|
||||
``' AND '``, producing strings like ``(expr1) AND (expr2)``. This
|
||||
reverses that: split on the ``)\\s+AND\\s+(`` boundary (case-insensitive,
|
||||
tolerating whitespace variations), strip the outer parens, and return the
|
||||
raw expressions.
|
||||
"""
|
||||
if not composed:
|
||||
return []
|
||||
# Unbalanced parens can't be a valid composed clause — fail closed so
|
||||
# the malformed string lands in the allowed-set check as-is (→ 403)
|
||||
# instead of splitting into fragments that might individually pass.
|
||||
if composed.count("(") != composed.count(")"):
|
||||
return [composed]
|
||||
raw = re.split(r"\)\s+AND\s+\(", composed, flags=re.IGNORECASE)
|
||||
# Strip exactly one outer paren added by _sanitize_clause.
|
||||
if raw[0].startswith("("):
|
||||
raw[0] = raw[0][1:]
|
||||
if raw[-1].endswith(")"):
|
||||
raw[-1] = raw[-1][:-1]
|
||||
# _sanitize_clause appends ``\n`` inside the parens when the expression
|
||||
# contains ``--`` (to terminate a trailing line comment). Strip it so
|
||||
# the result matches the stored raw expression.
|
||||
return [expr.rstrip("\n") for expr in raw]
|
||||
|
||||
|
||||
def _add_allowed_sql_from_query_context(
|
||||
extras_allowed: set[str],
|
||||
col_allowed: set[str],
|
||||
stored_query_context: dict[str, Any],
|
||||
) -> None:
|
||||
"""Add allowed SQL expressions from a stored query context."""
|
||||
for query in stored_query_context.get("queries") or []:
|
||||
for param in ("where", "having"):
|
||||
composed = (query.get("extras") or {}).get(param, "")
|
||||
for expr in _split_extras_clauses(composed):
|
||||
extras_allowed.add(expr)
|
||||
# Keep the full composed value as a fallback in case a stored
|
||||
# expression contains a literal ") AND (" that the split would
|
||||
# incorrectly break apart.
|
||||
if composed:
|
||||
extras_allowed.add(composed)
|
||||
for key in ("columns", "groupby"):
|
||||
for col in query.get(key) or []:
|
||||
if isinstance(col, dict) and col.get("sqlExpression"):
|
||||
col_allowed.add(col["sqlExpression"])
|
||||
|
||||
|
||||
def _collect_allowed_sql(
|
||||
stored_chart: "Slice",
|
||||
stored_query_context: Optional[dict[str, Any]],
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Collect the SQL expressions a guest user is allowed to send.
|
||||
|
||||
Returns ``(extras_allowed, col_allowed)``:
|
||||
|
||||
* ``extras_allowed`` — for validating ``extras.where``/``extras.having``:
|
||||
adhoc-filter SQL, legacy ``where`` param, stored query-context extras,
|
||||
and the ``1 = 0`` empty-filter sentinel.
|
||||
* ``col_allowed`` — for validating structured-filter ``col.sqlExpression``:
|
||||
everything in ``extras_allowed`` plus column SQL expressions from the
|
||||
chart's dimensions (which cross-filters legitimately reference).
|
||||
"""
|
||||
extras_allowed: set[str] = {_EMPTY_FILTER_SENTINEL}
|
||||
params = stored_chart.params_dict
|
||||
|
||||
for flt in params.get("adhoc_filters") or []:
|
||||
if (
|
||||
isinstance(flt, dict)
|
||||
and flt.get("expressionType") == "SQL"
|
||||
and flt.get("sqlExpression")
|
||||
):
|
||||
extras_allowed.add(flt["sqlExpression"])
|
||||
|
||||
if params.get("where"):
|
||||
extras_allowed.add(params["where"])
|
||||
|
||||
# Column expressions go only into col_allowed — they must not be
|
||||
# injectable as WHERE/HAVING predicates.
|
||||
col_allowed: set[str] = set(extras_allowed)
|
||||
_add_column_sql_expressions(col_allowed, params)
|
||||
|
||||
if stored_query_context:
|
||||
_add_allowed_sql_from_query_context(
|
||||
extras_allowed, col_allowed, stored_query_context
|
||||
)
|
||||
|
||||
return extras_allowed, col_allowed
|
||||
|
||||
|
||||
def _add_column_sql_expressions(target: set[str], params: dict[str, Any]) -> None:
|
||||
"""Add ``sqlExpression`` values from column params to *target*.
|
||||
|
||||
Handles both list-valued controls (``columns``, ``groupby``) and
|
||||
scalar-valued ones (``x_axis``, ``entity``, etc.).
|
||||
"""
|
||||
for key in _STORED_COLUMN_PARAMS:
|
||||
value = params.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
items = value if isinstance(value, (list, tuple)) else [value]
|
||||
for col in items:
|
||||
if isinstance(col, dict) and col.get("sqlExpression"):
|
||||
target.add(col["sqlExpression"])
|
||||
|
||||
|
||||
def _query_has_novel_extras(query: Any, allowed: set[str]) -> bool:
|
||||
"""Whether a query has novel ``extras.where``/``extras.having`` SQL.
|
||||
|
||||
The full composed value is checked first; if it is in ``allowed`` (which
|
||||
includes full composed values from the stored query context as a fallback)
|
||||
the split is skipped. If a stored expression contains a literal
|
||||
``) AND (`` the split may break it into fragments that fail individually —
|
||||
a false positive (403) rather than a bypass, and an acceptable trade-off.
|
||||
"""
|
||||
extras = getattr(query, "extras", None) or {}
|
||||
for param in ("where", "having"):
|
||||
composed = extras.get(param, "")
|
||||
if composed and composed not in allowed:
|
||||
for expr in _split_extras_clauses(composed):
|
||||
if expr not in allowed:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _query_has_novel_filter_col(query: Any, allowed: set[str]) -> bool:
|
||||
"""Whether a query has a structured filter ``col`` not in the allowed set.
|
||||
|
||||
Unlike ``_query_has_novel_extras`` this only checks the ``filter[].col``
|
||||
vector — the cross-filter path — and intentionally ignores
|
||||
``extras.where``/``extras.having``. Used for the scoped re-check after
|
||||
expanding ``allowed`` with sibling dashboard chart expressions: those
|
||||
borrowed expressions must only legitimize filter columns, not become
|
||||
injectable as arbitrary WHERE/HAVING predicates.
|
||||
"""
|
||||
for flt in getattr(query, "filter", None) or []:
|
||||
if isinstance(flt, dict):
|
||||
col = flt.get("col")
|
||||
if isinstance(col, dict) and col.get("sqlExpression"):
|
||||
if col["sqlExpression"] not in allowed:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _add_dashboard_column_expressions(
|
||||
allowed: set[str], dashboard_id: Any, target_chart_id: int
|
||||
) -> None:
|
||||
"""
|
||||
Add ``sqlExpression`` values from adhoc columns on every chart of the
|
||||
given dashboard (except the target chart, which is already covered).
|
||||
|
||||
This allows cross-filter structured filters whose ``col`` carries the
|
||||
source chart's custom SQL dimension to pass validation. Called lazily
|
||||
(only when an unrecognized adhoc SQL col is found) to avoid a DB query
|
||||
on the common path.
|
||||
|
||||
The dashboard is authorized via ``has_guest_access`` and the target chart
|
||||
must belong to the dashboard; otherwise no expressions are added.
|
||||
"""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from superset import db, security_manager
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
try:
|
||||
dashboard_id = int(dashboard_id)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
dashboard = (
|
||||
db.session.query(Dashboard).filter(Dashboard.id == dashboard_id).one_or_none()
|
||||
)
|
||||
if dashboard is None:
|
||||
return
|
||||
|
||||
if not security_manager.has_guest_access(dashboard):
|
||||
return
|
||||
|
||||
slice_ids = {s.id for s in dashboard.slices}
|
||||
if target_chart_id not in slice_ids:
|
||||
return
|
||||
|
||||
for slc in dashboard.slices:
|
||||
if slc.id == target_chart_id:
|
||||
continue
|
||||
_add_column_sql_expressions(allowed, slc.params_dict)
|
||||
|
||||
|
||||
def _sql_filters_modified(
|
||||
query_context: "QueryContext",
|
||||
form_data: dict[str, Any],
|
||||
stored_chart: "Slice",
|
||||
stored_query_context: Optional[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""
|
||||
Whether the request injects custom SQL not present on the stored chart.
|
||||
|
||||
Covers three vectors:
|
||||
|
||||
1. ``extras.where`` / ``extras.having`` — raw SQL strings.
|
||||
2. Adhoc filters with ``expressionType == "SQL"`` in ``form_data``.
|
||||
3. Structured ``{col, op, val}`` filters whose ``col`` carries a
|
||||
``sqlExpression`` (reaches ``adhoc_column_to_sqla``).
|
||||
|
||||
The ``(1 = 0)`` empty-filter sentinel injected by required-but-empty
|
||||
native Select filters is always allowed. For vector 3, SQL expressions
|
||||
from all charts on the requesting dashboard are allowed so that
|
||||
cross-filters referencing a sibling chart's custom SQL dimension pass.
|
||||
|
||||
Cache-replay requests (``/data/<cache_key>``) are skipped: the original
|
||||
request already passed the full check, and ``_sanitize_filters`` may have
|
||||
rewritten ``extras`` in place before caching (comment normalization,
|
||||
Jinja rendering), making byte-equality comparison unreliable.
|
||||
"""
|
||||
if getattr(query_context, "_from_cache_replay", False) is True:
|
||||
return False
|
||||
|
||||
extras_allowed, col_allowed = _collect_allowed_sql(
|
||||
stored_chart, stored_query_context
|
||||
)
|
||||
|
||||
# Vector 1: extras.where / extras.having
|
||||
if any(_query_has_novel_extras(q, extras_allowed) for q in query_context.queries):
|
||||
return True
|
||||
|
||||
# Vector 3: structured filter col with adhoc SQL.
|
||||
# Sibling chart column expressions (cross-filter) are allowed; the
|
||||
# dashboard lookup is deferred so the common case pays no DB cost.
|
||||
if any(_query_has_novel_filter_col(q, col_allowed) for q in query_context.queries):
|
||||
if dashboard_id := (form_data or {}).get("dashboardId"):
|
||||
_add_dashboard_column_expressions(
|
||||
col_allowed, dashboard_id, stored_chart.id
|
||||
)
|
||||
if any(
|
||||
_query_has_novel_filter_col(q, col_allowed) for q in query_context.queries
|
||||
):
|
||||
return True
|
||||
|
||||
# Vector 2: SQL adhoc filters in form_data
|
||||
stored_sql_filters: set[str] = {
|
||||
freeze_value(flt)
|
||||
for flt in stored_chart.params_dict.get("adhoc_filters") or []
|
||||
if isinstance(flt, dict) and flt.get("expressionType") == "SQL"
|
||||
}
|
||||
|
||||
for flt in form_data.get("adhoc_filters") or []:
|
||||
if not isinstance(flt, dict):
|
||||
continue
|
||||
if flt.get("expressionType") == "SQL":
|
||||
if freeze_value(flt) not in stored_sql_filters:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
#: Chart params keys that hold the metrics a chart renders. Different chart
|
||||
#: types store their metrics under control-specific keys (``metric`` for
|
||||
#: big number, ``x``/``y``/``size`` for bubble, and so on); a guest requesting
|
||||
@@ -1576,13 +1306,6 @@ def query_context_modified(query_context: "QueryContext") -> bool:
|
||||
# than accepting any payload, constrain them to the column(s) the dashboard's
|
||||
# native filter is allowed to target; other chartless paths keep prior
|
||||
# behavior (see _native_filter_request_modified).
|
||||
#
|
||||
# SQL extras (extras.where/having) are NOT validated on chartless paths:
|
||||
# without a stored chart there is nothing to validate against, and
|
||||
# tightening this would break legitimate chartless flows (native-filter
|
||||
# pre-filtering, drill-to-detail) that carry SQL extras. These paths
|
||||
# are still protected by datasource-access checks in raise_for_access.
|
||||
# The _sql_filters_modified check below covers chart payloads only.
|
||||
if stored_chart is None:
|
||||
return _native_filter_request_modified(query_context)
|
||||
|
||||
@@ -1652,19 +1375,6 @@ def query_context_modified(query_context: "QueryContext") -> bool:
|
||||
)
|
||||
return True
|
||||
|
||||
# SQL predicates (extras.where/having, SQL adhoc filters) must match
|
||||
# what was saved on the chart; injected custom SQL is rejected.
|
||||
if _sql_filters_modified(
|
||||
query_context, form_data, stored_chart, stored_query_context
|
||||
):
|
||||
logger.warning(
|
||||
"Guest chart payload rejected for slice %s: SQL filter/extras "
|
||||
"not on the stored chart (stored query_context %s)",
|
||||
stored_chart.id,
|
||||
stored_context_state,
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
|
||||
+2
-16
@@ -176,8 +176,6 @@ METRIC_MAP_TYPE = {
|
||||
"PERCENTILE": "floating",
|
||||
"VARIANCE": "floating",
|
||||
"STDDEV": "floating",
|
||||
"STDDEV_SAMP": "floating",
|
||||
"VAR_SAMP": "floating",
|
||||
}
|
||||
|
||||
|
||||
@@ -186,15 +184,6 @@ class AdhocMetricExpressionType(StrEnum):
|
||||
SQL = "SQL"
|
||||
|
||||
|
||||
# Aggregates with no safe, universal cross-dialect spelling -- unlike
|
||||
# SUM/COUNT/AVG/MIN/MAX/COUNT_DISTINCT, whose SQL is generated the same way on
|
||||
# every engine. Support for these is opt-in per `BaseEngineSpec` (see
|
||||
# `get_extended_aggregation_func`); used to distinguish a genuinely invalid
|
||||
# aggregate name from one that is valid but unsupported on the current database,
|
||||
# for a clearer user-facing error.
|
||||
EXTENDED_METRIC_AGGREGATES = frozenset({"MEDIAN", "STDDEV_SAMP", "VAR_SAMP"})
|
||||
|
||||
|
||||
class SqlExpressionType(StrEnum):
|
||||
"""Types of SQL expressions that can be validated."""
|
||||
|
||||
@@ -1831,14 +1820,11 @@ def get_metric_type_from_column(column: Any, datasource: Explorable) -> str:
|
||||
expression: str = metric.expression
|
||||
|
||||
match = re.match(
|
||||
r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX|FIRST|LAST"
|
||||
r"|MEDIAN|STDDEV_SAMP|VAR_SAMP)\s*\((.*)\)",
|
||||
expression,
|
||||
re.IGNORECASE,
|
||||
r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX|FIRST|LAST)\((.*)\)", expression
|
||||
)
|
||||
|
||||
if match:
|
||||
operation = match.group(1).upper()
|
||||
operation = match.group(1)
|
||||
return METRIC_MAP_TYPE.get(operation, "")
|
||||
|
||||
logger.debug("Unexpected metric expression type: %s", expression)
|
||||
|
||||
@@ -22,7 +22,6 @@ from marshmallow import ValidationError
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from superset.charts.schemas import (
|
||||
ChartDataAdhocMetricSchema,
|
||||
ChartDataExtrasSchema,
|
||||
ChartDataPostProcessingOperationSchema,
|
||||
ChartDataProphetOptionsSchema,
|
||||
@@ -539,23 +538,3 @@ def test_post_processing_extra_op_is_accepted(app_context: None) -> None:
|
||||
schema = ChartDataPostProcessingOperationSchema()
|
||||
|
||||
assert schema.load({"operation": "_custom_op"})["operation"] == "_custom_op"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
|
||||
def test_chart_data_adhoc_metric_schema_accepts_extended_aggregates(
|
||||
app_context: None, aggregate: str
|
||||
) -> None:
|
||||
"""
|
||||
The chart-data REST schema's ``aggregate`` enum must stay in sync with
|
||||
``EXTENDED_METRIC_AGGREGATES``, otherwise Swagger/generated clients
|
||||
reject requests using these compiler-supported aggregates.
|
||||
"""
|
||||
schema = ChartDataAdhocMetricSchema()
|
||||
result = schema.load(
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"aggregate": aggregate,
|
||||
"column": {"column_name": "value"},
|
||||
}
|
||||
)
|
||||
assert result["aggregate"] == aggregate
|
||||
|
||||
@@ -18,11 +18,15 @@ from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from marshmallow import ValidationError
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from superset.commands.dataset.create import CreateDatasetCommand
|
||||
from superset.commands.dataset.exceptions import DatasetInvalidError
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
from superset.exceptions import SupersetParseError
|
||||
from superset.exceptions import (
|
||||
SupersetGenericDBErrorException,
|
||||
SupersetParseError,
|
||||
)
|
||||
from superset.models.core import Database
|
||||
|
||||
|
||||
@@ -250,3 +254,83 @@ def test_create_dataset_generic_exists_error_when_no_twin() -> None:
|
||||
)
|
||||
with pytest.raises(DatasetInvalidError):
|
||||
command.validate()
|
||||
|
||||
|
||||
def test_create_dataset_metadata_fetch_error_is_structured(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""A metadata-fetch failure must surface the engine's own message.
|
||||
|
||||
``run()`` executes the SQL to introspect columns; the resulting
|
||||
``SupersetGenericDBErrorException`` used to escape as a 500 "Fatal error".
|
||||
"""
|
||||
mocker.patch.object(CreateDatasetCommand, "validate")
|
||||
dataset = Mock()
|
||||
dataset.fetch_metadata.side_effect = SupersetGenericDBErrorException(
|
||||
message="Invalid SQL: Unable to parse: SELECT ...",
|
||||
)
|
||||
mocker.patch(
|
||||
"superset.commands.dataset.create.DatasetDAO.create",
|
||||
return_value=dataset,
|
||||
)
|
||||
|
||||
command = CreateDatasetCommand(
|
||||
{
|
||||
"database": 1,
|
||||
"table_name": "dataset wrong",
|
||||
"sql": "SELECT ...",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(DatasetInvalidError) as exc_info:
|
||||
command.run()
|
||||
|
||||
validation_errors = exc_info.value._exceptions
|
||||
assert len(validation_errors) == 1
|
||||
assert validation_errors[0].field_name == "sql"
|
||||
assert "Invalid SQL: Unable to parse: SELECT ..." in str(
|
||||
validation_errors[0].messages[0]
|
||||
)
|
||||
|
||||
|
||||
def test_create_dataset_metadata_fetch_error_physical_table(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""The same conversion applies to physical datasets, keyed on ``table``."""
|
||||
mocker.patch.object(CreateDatasetCommand, "validate")
|
||||
dataset = Mock()
|
||||
dataset.fetch_metadata.side_effect = SupersetGenericDBErrorException(
|
||||
message="(psycopg2.OperationalError) could not connect to server",
|
||||
)
|
||||
mocker.patch(
|
||||
"superset.commands.dataset.create.DatasetDAO.create",
|
||||
return_value=dataset,
|
||||
)
|
||||
|
||||
command = CreateDatasetCommand({"database": 1, "table_name": "physical_table"})
|
||||
|
||||
with pytest.raises(DatasetInvalidError) as exc_info:
|
||||
command.run()
|
||||
|
||||
validation_errors = exc_info.value._exceptions
|
||||
assert validation_errors[0].field_name == "table"
|
||||
assert "could not connect to server" in str(validation_errors[0].messages[0])
|
||||
|
||||
|
||||
def test_create_dataset_run_succeeds_when_metadata_fetch_works(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Control: the happy path still returns the created dataset."""
|
||||
mocker.patch.object(CreateDatasetCommand, "validate")
|
||||
dataset = Mock()
|
||||
mocker.patch(
|
||||
"superset.commands.dataset.create.DatasetDAO.create",
|
||||
return_value=dataset,
|
||||
)
|
||||
|
||||
command = CreateDatasetCommand(
|
||||
{"database": 1, "table_name": "good_dataset", "sql": "SELECT 1 AS a"}
|
||||
)
|
||||
|
||||
assert command.run() is dataset
|
||||
dataset.fetch_metadata.assert_called_once()
|
||||
|
||||
@@ -214,3 +214,46 @@ def test_handle_filters_args_returns_request_scoped_filters(
|
||||
fresh_filters = api.datamodel.get_filters.return_value
|
||||
assert fresh_filters.rest_add_filters.call_count == 2
|
||||
assert fresh_filters.get_joined_filters.call_count == 2
|
||||
|
||||
|
||||
def test_post_dataset_with_invalid_sql_returns_actionable_422(
|
||||
session: Session,
|
||||
client: Any,
|
||||
full_api_access: None,
|
||||
) -> None:
|
||||
"""Saving a dataset over unrunnable SQL must explain what is wrong.
|
||||
|
||||
With blanket database access ``validate()`` never parses the SQL, so
|
||||
``run()``'s column introspection is the first thing to reject it. That
|
||||
used to surface as a bare 500 ``{"message": "Fatal error"}``.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
from superset.models.core import Database
|
||||
|
||||
SqlaTable.metadata.create_all(db.session.get_bind())
|
||||
|
||||
database = Database(database_name="invalid_sql_db", sqlalchemy_uri="sqlite://")
|
||||
db.session.add(database)
|
||||
db.session.flush()
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/dataset/",
|
||||
json={
|
||||
"database": database.id,
|
||||
"schema": "main",
|
||||
"table_name": "dataset wrong",
|
||||
"sql": "SELECT ...",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
message = response.json["message"]
|
||||
assert "Fatal error" not in str(message)
|
||||
# Not the parser's exact wording -- that would break on a sqlglot bump.
|
||||
assert message["sql"][0].startswith("Invalid SQL")
|
||||
|
||||
# The failed create must not leave a half-built dataset behind.
|
||||
assert (
|
||||
db.session.query(SqlaTable).filter_by(table_name="dataset wrong").one_or_none()
|
||||
is None
|
||||
)
|
||||
|
||||
@@ -301,43 +301,6 @@ def test_aurora_mysql_update_params_from_encrypted_extra_with_iam() -> None:
|
||||
# SSL should be configured via the database's extra settings.
|
||||
|
||||
|
||||
def test_aurora_postgres_inherits_extended_aggregations() -> None:
|
||||
"""
|
||||
Aurora PostgreSQL (and its Data API variant) is AWS's wire- and
|
||||
SQL-compatible managed Postgres, not a forked query engine, so it
|
||||
inherits `_extended_aggregations` from `PostgresBaseEngineSpec`
|
||||
unmodified -- see the comment above that dict.
|
||||
"""
|
||||
from superset.db_engine_specs.aurora import (
|
||||
AuroraPostgresDataAPI,
|
||||
AuroraPostgresEngineSpec,
|
||||
)
|
||||
|
||||
for spec in (AuroraPostgresEngineSpec, AuroraPostgresDataAPI):
|
||||
assert spec.get_extended_aggregation_func("MEDIAN") is not None
|
||||
assert spec.get_extended_aggregation_func("STDDEV_SAMP") is not None
|
||||
assert spec.get_extended_aggregation_func("VAR_SAMP") is not None
|
||||
|
||||
|
||||
def test_aurora_mysql_inherits_extended_aggregations() -> None:
|
||||
"""
|
||||
Aurora MySQL (and its Data API variant) is AWS's wire- and
|
||||
SQL-compatible managed MySQL, not a forked query engine, so it inherits
|
||||
`_extended_aggregations` from `MySQLEngineSpec` unmodified -- see the
|
||||
comment above that dict.
|
||||
"""
|
||||
from superset.db_engine_specs.aurora import (
|
||||
AuroraMySQLDataAPI,
|
||||
AuroraMySQLEngineSpec,
|
||||
)
|
||||
|
||||
for spec in (AuroraMySQLEngineSpec, AuroraMySQLDataAPI):
|
||||
assert spec.get_extended_aggregation_func("STDDEV_SAMP") is not None
|
||||
assert spec.get_extended_aggregation_func("VAR_SAMP") is not None
|
||||
# Same as MySQL, MEDIAN is not supported.
|
||||
assert spec.get_extended_aggregation_func("MEDIAN") is None
|
||||
|
||||
|
||||
def test_aurora_data_api_classes_unchanged() -> None:
|
||||
from superset.db_engine_specs.aurora import (
|
||||
AuroraMySQLDataAPI,
|
||||
|
||||
@@ -1508,22 +1508,3 @@ def test_array_capabilities_raise_when_unsupported(method: str) -> None:
|
||||
args = (column("c"), ["v"]) if "contains" in method else (column("c"),)
|
||||
with pytest.raises(NotImplementedError):
|
||||
fn(*args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
|
||||
def test_base_spec_extended_aggregation_func_defaults_to_unsupported(
|
||||
aggregate: str,
|
||||
) -> None:
|
||||
"""
|
||||
By default, an engine spec has no verified expression for the "extended"
|
||||
aggregates (MEDIAN/STDDEV_SAMP/VAR_SAMP) -- they must be explicitly opted
|
||||
into per engine spec, the same way `supports_grouping_sets` and
|
||||
`_time_grain_expressions` work. Silence here means "unsupported", not
|
||||
"untested" -- callers must not fall back to guessing SQL.
|
||||
"""
|
||||
assert BaseEngineSpec.get_extended_aggregation_func(aggregate) is None
|
||||
|
||||
|
||||
def test_base_spec_extended_aggregation_func_unknown_name_is_unsupported() -> None:
|
||||
"""An aggregate name outside the known extended set is also just None."""
|
||||
assert BaseEngineSpec.get_extended_aggregation_func("NOT_A_REAL_AGGREGATE") is None
|
||||
|
||||
@@ -191,62 +191,3 @@ def test_fetch_data_preserves_cursor_description(mocker: MockerFixture) -> None:
|
||||
assert [col[0] for col in cursor.description] == ["col1", "col2"]
|
||||
finally:
|
||||
raw_conn.close()
|
||||
|
||||
|
||||
def test_extended_aggregation_func_median_stddev_var_compiles() -> None:
|
||||
"""
|
||||
MEDIAN/STDDEV_SAMP/VAR_SAMP compile to the expected DuckDB SQL function
|
||||
calls. See `test_extended_aggregation_func_median_stddev_var_executes`
|
||||
for verification against a live in-process DuckDB instance.
|
||||
"""
|
||||
from sqlalchemy import column
|
||||
|
||||
from superset.db_engine_specs.duckdb import DuckDBEngineSpec
|
||||
|
||||
col = column("sales")
|
||||
|
||||
for aggregate, expected_sql in [
|
||||
("MEDIAN", "median(sales)"),
|
||||
("STDDEV_SAMP", "stddev_samp(sales)"),
|
||||
("VAR_SAMP", "var_samp(sales)"),
|
||||
]:
|
||||
func = DuckDBEngineSpec.get_extended_aggregation_func(aggregate)
|
||||
assert func is not None
|
||||
compiled = str(func(col).compile(compile_kwargs={"literal_binds": True}))
|
||||
assert compiled == expected_sql
|
||||
|
||||
|
||||
def test_extended_aggregation_func_median_stddev_var_executes() -> None:
|
||||
"""
|
||||
MEDIAN/STDDEV_SAMP/VAR_SAMP execute against a live in-process DuckDB
|
||||
instance and return values matching Python's `statistics` module
|
||||
(sample standard deviation/variance) for the same input.
|
||||
"""
|
||||
import statistics
|
||||
|
||||
from sqlalchemy import create_engine, literal_column, select, text
|
||||
|
||||
from superset.db_engine_specs.duckdb import DuckDBEngineSpec
|
||||
|
||||
values = [1.0, 2.0, 4.0, 8.0, 16.0]
|
||||
|
||||
engine = create_engine("duckdb:///:memory:")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("CREATE TABLE t (sales DOUBLE)"))
|
||||
conn.execute(
|
||||
text("INSERT INTO t VALUES (:sales)"),
|
||||
[{"sales": v} for v in values],
|
||||
)
|
||||
|
||||
expected = {
|
||||
"MEDIAN": statistics.median(values),
|
||||
"STDDEV_SAMP": statistics.stdev(values),
|
||||
"VAR_SAMP": statistics.variance(values),
|
||||
}
|
||||
|
||||
for aggregate, expected_value in expected.items():
|
||||
func = DuckDBEngineSpec.get_extended_aggregation_func(aggregate)
|
||||
assert func is not None
|
||||
query = select(func(literal_column("sales"))).select_from(text("t"))
|
||||
result = conn.execute(query).scalar()
|
||||
assert result == pytest.approx(expected_value)
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
`_extended_aggregations` (MEDIAN/STDDEV_SAMP/VAR_SAMP) is verified against
|
||||
real engine behavior before being enabled -- see `BaseEngineSpec` for the
|
||||
rationale. `PostgresBaseEngineSpec`/`PostgresEngineSpec`/`MySQLEngineSpec`
|
||||
subclasses that share SQL dialect helpers with Postgres/MySQL but run a
|
||||
materially different query engine (a proprietary appliance, a distributed
|
||||
SQL engine, or an OLAP engine) must not silently inherit that dict: each
|
||||
verified engine spec opts in explicitly, everything else stays unsupported
|
||||
until someone verifies it against a live instance.
|
||||
"""
|
||||
|
||||
from typing import Type
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.db_engine_specs.cockroachdb import CockroachDbEngineSpec
|
||||
from superset.db_engine_specs.doris import DorisEngineSpec
|
||||
from superset.db_engine_specs.greenplum import GreenplumEngineSpec
|
||||
from superset.db_engine_specs.hana import HanaEngineSpec
|
||||
from superset.db_engine_specs.hologres import HologresEngineSpec
|
||||
from superset.db_engine_specs.netezza import NetezzaEngineSpec
|
||||
from superset.db_engine_specs.oceanbase import OceanBaseEngineSpec
|
||||
from superset.db_engine_specs.risingwave import RisingWaveDbEngineSpec
|
||||
from superset.db_engine_specs.snowflake import SnowflakeEngineSpec
|
||||
from superset.db_engine_specs.starrocks import StarRocksEngineSpec
|
||||
from superset.db_engine_specs.vertica import VerticaEngineSpec
|
||||
from superset.db_engine_specs.yugabytedb import YugabyteDBEngineSpec
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec_cls",
|
||||
[
|
||||
VerticaEngineSpec,
|
||||
NetezzaEngineSpec,
|
||||
HanaEngineSpec,
|
||||
SnowflakeEngineSpec,
|
||||
CockroachDbEngineSpec,
|
||||
GreenplumEngineSpec,
|
||||
RisingWaveDbEngineSpec,
|
||||
YugabyteDBEngineSpec,
|
||||
HologresEngineSpec,
|
||||
DorisEngineSpec,
|
||||
StarRocksEngineSpec,
|
||||
OceanBaseEngineSpec,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("aggregate", ["MEDIAN", "STDDEV_SAMP", "VAR_SAMP"])
|
||||
def test_unverified_postgres_and_mysql_family_specs_reject_extended_aggregations(
|
||||
spec_cls: Type[BaseEngineSpec], aggregate: str
|
||||
) -> None:
|
||||
assert spec_cls.get_extended_aggregation_func(aggregate) is None
|
||||
@@ -1,35 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from superset.db_engine_specs.mariadb import MariaDBEngineSpec
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
|
||||
def test_mariadb_inherits_from_mysql() -> None:
|
||||
assert issubclass(MariaDBEngineSpec, MySQLEngineSpec)
|
||||
|
||||
|
||||
def test_mariadb_inherits_extended_aggregations() -> None:
|
||||
"""
|
||||
MariaDB is a MySQL fork implementing the same aggregate functions, not a
|
||||
materially different query engine, so it inherits `_extended_aggregations`
|
||||
from `MySQLEngineSpec` unmodified -- see the comment above that dict.
|
||||
"""
|
||||
assert MariaDBEngineSpec.get_extended_aggregation_func("STDDEV_SAMP") is not None
|
||||
assert MariaDBEngineSpec.get_extended_aggregation_func("VAR_SAMP") is not None
|
||||
# Same as MySQL, MEDIAN is not supported.
|
||||
assert MariaDBEngineSpec.get_extended_aggregation_func("MEDIAN") is None
|
||||
@@ -401,39 +401,3 @@ def test_identifier_quote_uses_backticks() -> None:
|
||||
"end": "`",
|
||||
"escape_by_doubling": True,
|
||||
}
|
||||
|
||||
|
||||
def test_extended_aggregation_func_stddev_var_sample() -> None:
|
||||
"""
|
||||
Verified against a live mysql:8.0 instance, including under GROUP BY ...
|
||||
WITH ROLLUP: these produce the correct database-wide sample statistic, and
|
||||
match the same-input results from postgres/duckdb exactly.
|
||||
"""
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
|
||||
|
||||
col = column("sales")
|
||||
|
||||
stddev_expr = spec.get_extended_aggregation_func("STDDEV_SAMP")
|
||||
assert stddev_expr is not None
|
||||
assert (
|
||||
str(stddev_expr(col).compile(compile_kwargs={"literal_binds": True}))
|
||||
== "stddev_samp(sales)"
|
||||
)
|
||||
|
||||
var_expr = spec.get_extended_aggregation_func("VAR_SAMP")
|
||||
assert var_expr is not None
|
||||
assert (
|
||||
str(var_expr(col).compile(compile_kwargs={"literal_binds": True}))
|
||||
== "var_samp(sales)"
|
||||
)
|
||||
|
||||
|
||||
def test_extended_aggregation_func_median_unsupported() -> None:
|
||||
"""
|
||||
MySQL has neither a MEDIAN function nor PERCENTILE_CONT (confirmed against
|
||||
a live mysql:8.0 instance: both error). Must not silently fall back to
|
||||
a guessed expression.
|
||||
"""
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
|
||||
|
||||
assert spec.get_extended_aggregation_func("MEDIAN") is None
|
||||
|
||||
@@ -501,30 +501,3 @@ def test_get_schema_names_excludes_only_actual_system_schemas(
|
||||
"pgstats",
|
||||
"information_schema",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("aggregate", "expected_sql"),
|
||||
[
|
||||
("MEDIAN", "percentile_cont(0.5) WITHIN GROUP (ORDER BY sales)"),
|
||||
("STDDEV_SAMP", "stddev_samp(sales)"),
|
||||
("VAR_SAMP", "var_samp(sales)"),
|
||||
],
|
||||
)
|
||||
def test_extended_aggregation_func_compiles_expected_sql(
|
||||
aggregate: str, expected_sql: str
|
||||
) -> None:
|
||||
"""
|
||||
Verified against a live postgres:16 instance (including under GROUPING
|
||||
SETS): these expressions compute the correct database-wide statistic, not
|
||||
an aggregate-of-per-group-aggregates.
|
||||
"""
|
||||
func = spec.get_extended_aggregation_func(aggregate)
|
||||
assert func is not None
|
||||
|
||||
compiled = str(
|
||||
func(column("sales")).compile(
|
||||
dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}
|
||||
)
|
||||
)
|
||||
assert compiled == expected_sql
|
||||
|
||||
@@ -79,33 +79,3 @@ def test_normalize_table_name_for_upload(
|
||||
|
||||
assert normalized_table == expected_table
|
||||
assert normalized_schema == expected_schema
|
||||
|
||||
|
||||
def test_extended_aggregation_func_inherited_from_postgres() -> None:
|
||||
"""
|
||||
Redshift is a Postgres fork and inherits STDDEV_SAMP/VAR_SAMP from
|
||||
`PostgresBaseEngineSpec` -- its documented SQL function reference matches
|
||||
Postgres for these functions, though this has not been separately
|
||||
verified against a live Redshift instance (see the SIP doc for caveats).
|
||||
|
||||
MEDIAN is overridden rather than inherited (see
|
||||
`RedshiftEngineSpec._extended_aggregations`): unlike Postgres, Redshift
|
||||
documents a native `MEDIAN(x)` function, so Redshift emits that directly
|
||||
instead of Postgres's `percentile_cont(0.5) WITHIN GROUP (ORDER BY col)`
|
||||
spelling -- which also sidesteps Redshift's documented restriction
|
||||
against more than one sort-based aggregate with a different ORDER BY in
|
||||
the same query, e.g. `MEDIAN(sales)` alongside `MEDIAN(margin)`.
|
||||
"""
|
||||
from sqlalchemy import column
|
||||
|
||||
col = column("sales")
|
||||
|
||||
median_func = RedshiftEngineSpec.get_extended_aggregation_func("MEDIAN")
|
||||
assert median_func is not None
|
||||
assert (
|
||||
str(median_func(col).compile(compile_kwargs={"literal_binds": True}))
|
||||
== "median(sales)"
|
||||
)
|
||||
|
||||
for aggregate in ("STDDEV_SAMP", "VAR_SAMP"):
|
||||
assert RedshiftEngineSpec.get_extended_aggregation_func(aggregate) is not None
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
from superset.db_engine_specs.timescaledb import TimescaleDBEngineSpec
|
||||
|
||||
|
||||
def test_timescaledb_inherits_from_postgres_base() -> None:
|
||||
assert issubclass(TimescaleDBEngineSpec, PostgresBaseEngineSpec)
|
||||
|
||||
|
||||
def test_timescaledb_inherits_extended_aggregations() -> None:
|
||||
"""
|
||||
TimescaleDB is a Postgres extension, not a forked query engine -- it runs
|
||||
unmodified Postgres aggregate execution -- so it inherits
|
||||
`_extended_aggregations` from `PostgresBaseEngineSpec` unmodified -- see
|
||||
the comment above that dict.
|
||||
"""
|
||||
assert TimescaleDBEngineSpec.get_extended_aggregation_func("MEDIAN") is not None
|
||||
assert (
|
||||
TimescaleDBEngineSpec.get_extended_aggregation_func("STDDEV_SAMP") is not None
|
||||
)
|
||||
assert TimescaleDBEngineSpec.get_extended_aggregation_func("VAR_SAMP") is not None
|
||||
@@ -89,24 +89,6 @@ class TestCreateMetricObject:
|
||||
assert result["optionName"] == "metric_revenue"
|
||||
assert result["expressionType"] == "SIMPLE"
|
||||
|
||||
def test_create_metric_object_normalizes_stddev_var_shorthand(self) -> None:
|
||||
"""
|
||||
Pre-SIP "STDDEV"/"VAR" shorthand is normalized to the real,
|
||||
unambiguous aggregate names ("STDDEV_SAMP"/"VAR_SAMP") this chart
|
||||
can actually query -- see docs/sip/median-stddev-variance-aggregates.md.
|
||||
Before this, a chart built with "STDDEV"/"VAR" always errored at
|
||||
query time, since that name was never a real Superset aggregate.
|
||||
"""
|
||||
stddev_col = ColumnRef(name="price", aggregate="STDDEV", label="Std Dev")
|
||||
result = create_metric_object(stddev_col)
|
||||
assert isinstance(result, dict)
|
||||
assert result["aggregate"] == "STDDEV_SAMP"
|
||||
|
||||
var_col = ColumnRef(name="price", aggregate="VAR", label="Variance")
|
||||
result = create_metric_object(var_col)
|
||||
assert isinstance(result, dict)
|
||||
assert result["aggregate"] == "VAR_SAMP"
|
||||
|
||||
def test_create_metric_object_default_aggregate(self) -> None:
|
||||
"""Test creating metric object with default aggregate"""
|
||||
col = ColumnRef(name="orders")
|
||||
|
||||
@@ -2130,76 +2130,6 @@ def test_adhoc_metric_to_sqla_invalid_simple_aggregate_raises_validation_error(
|
||||
table.adhoc_metric_to_sqla(metric, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"aggregate,expected_substring",
|
||||
[
|
||||
("MEDIAN", "percentile_cont"),
|
||||
("STDDEV_SAMP", "stddev_samp"),
|
||||
("VAR_SAMP", "var_samp"),
|
||||
],
|
||||
)
|
||||
def test_adhoc_metric_to_sqla_extended_aggregate_on_supported_engine(
|
||||
aggregate: str,
|
||||
expected_substring: str,
|
||||
) -> None:
|
||||
"""
|
||||
MEDIAN/STDDEV_SAMP/VAR_SAMP compile correctly end-to-end on an engine that
|
||||
supports them (Postgres), via the same `adhoc_metric_to_sqla` path every
|
||||
other aggregate uses -- no pivot-table-specific code involved.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable, TableColumn
|
||||
from superset.models.core import Database
|
||||
|
||||
pg_database = Database(database_name="pg", sqlalchemy_uri="postgresql://u:p@h/d")
|
||||
table = SqlaTable(
|
||||
database=pg_database,
|
||||
schema=None,
|
||||
table_name="t",
|
||||
columns=[TableColumn(column_name="sales")],
|
||||
)
|
||||
metric: AdhocMetric = {
|
||||
"expressionType": "SIMPLE",
|
||||
"column": {"column_name": "sales"},
|
||||
"aggregate": aggregate,
|
||||
"label": f"{aggregate} sales",
|
||||
}
|
||||
|
||||
sqla_metric = table.adhoc_metric_to_sqla(metric, {})
|
||||
|
||||
assert expected_substring in str(sqla_metric).lower()
|
||||
|
||||
|
||||
def test_adhoc_metric_to_sqla_extended_aggregate_on_unsupported_engine_raises_specific_error( # noqa: E501
|
||||
database: Database,
|
||||
) -> None:
|
||||
"""
|
||||
A recognized extended aggregate (MEDIAN) that this engine (SQLite, via the
|
||||
`database` fixture) has no verified expression for raises a specific
|
||||
"not supported on this database" error, distinct from the generic
|
||||
"invalid aggregate" error a bogus aggregate name gets -- callers should be
|
||||
able to tell "this isn't a real thing" from "this engine can't do it"
|
||||
without emitting unverified SQL either way.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable, TableColumn
|
||||
from superset.exceptions import QueryObjectValidationError
|
||||
|
||||
table = SqlaTable(
|
||||
database=database,
|
||||
schema=None,
|
||||
table_name="t",
|
||||
columns=[TableColumn(column_name="a")],
|
||||
)
|
||||
metric: AdhocMetric = {
|
||||
"expressionType": "SIMPLE",
|
||||
"column": {"column_name": "a"},
|
||||
"aggregate": "MEDIAN",
|
||||
"label": "Median a",
|
||||
}
|
||||
|
||||
with pytest.raises(QueryObjectValidationError, match="not supported"):
|
||||
table.adhoc_metric_to_sqla(metric, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql_expression", [None, "", " "])
|
||||
def test_adhoc_metric_to_sqla_invalid_sql_expression_raises_validation_error(
|
||||
database: Database,
|
||||
|
||||
@@ -36,7 +36,6 @@ from superset.extensions import appbuilder
|
||||
from superset.models.slice import Slice
|
||||
from superset.security.manager import (
|
||||
_collect_sortable_identifiers,
|
||||
_sql_filters_modified,
|
||||
freeze_value,
|
||||
query_context_modified,
|
||||
SupersetSecurityManager,
|
||||
@@ -3792,683 +3791,3 @@ def test_validate_guest_token_resources_accepts_embedded_int_id(
|
||||
sm.validate_guest_token_resources(
|
||||
[{"type": GuestTokenResourceType.DASHBOARD, "id": 5}]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _sql_filters_modified – block custom SQL injection by guest users
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sql_filters_extras_where_injected_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Injecting extras.where when the chart has no SQL filters is blocked."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"metrics": ["count"]}
|
||||
|
||||
query = QueryObject(extras={"where": "1=1"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_extras_having_injected_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Injecting extras.having when the chart has no SQL filters is blocked."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(extras={"having": "COUNT(*) > 0"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_extras_where_replay_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Replaying the chart's own SQL WHERE filter is allowed."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "region = 'EMEA'",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
# freeform_where_having wraps each clause in parens
|
||||
query = QueryObject(extras={"where": "(region = 'EMEA')"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_extras_having_replay_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Replaying the chart's own SQL HAVING filter is allowed."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "SUM(sales) > 100",
|
||||
"clause": "HAVING",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
query = QueryObject(extras={"having": "(SUM(sales) > 100)"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_adhoc_sql_filter_injected_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Injecting a new SQL adhoc filter not on the stored chart is blocked."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject()
|
||||
query_context.queries = [query]
|
||||
|
||||
injected_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "1=1",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
form_data: dict[str, Any] = {"slice_id": 1, "adhoc_filters": [injected_filter]}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_adhoc_sql_filter_replay_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Replaying the exact stored SQL adhoc filter is allowed."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "region = 'EMEA'",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
query = QueryObject()
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1, "adhoc_filters": [sql_filter]}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_empty_extras_always_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""No SQL in extras is always allowed, even when the chart has SQL filters."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "region = 'EMEA'",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
query = QueryObject()
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_from_stored_qc_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""extras.where from stored query_context is allowed."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
stored_qc = {
|
||||
"queries": [{"extras": {"where": "(col > 5)"}}],
|
||||
}
|
||||
|
||||
query = QueryObject(extras={"where": "(col > 5)"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
|
||||
|
||||
|
||||
def test_sql_filters_multi_query_stored_predicate_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Multiple queries replaying predicates from the stored chart are allowed.
|
||||
|
||||
The allowed set is global across all stored queries — per-query pinning is
|
||||
intentionally not applied because there is no stable identity linking a
|
||||
request query to a stored query, and all queries share the same
|
||||
chart/datasource so predicates only restrict rows, never expand access.
|
||||
"""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
stored_qc = {
|
||||
"queries": [
|
||||
{"extras": {"where": "(region = 'EMEA')"}},
|
||||
{"extras": {"where": "(status = 'active')"}},
|
||||
],
|
||||
}
|
||||
|
||||
# Both request queries use predicates from the stored chart.
|
||||
query_context.queries = [
|
||||
QueryObject(extras={"where": "(region = 'EMEA')"}),
|
||||
QueryObject(extras={"where": "(status = 'active')"}),
|
||||
]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
|
||||
|
||||
|
||||
def test_sql_filters_multi_query_novel_predicate_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""A novel predicate on any query is blocked even when others are valid."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
stored_qc = {
|
||||
"queries": [{"extras": {"where": "(region = 'EMEA')"}}],
|
||||
}
|
||||
|
||||
query_context.queries = [
|
||||
QueryObject(extras={"where": "(region = 'EMEA')"}),
|
||||
QueryObject(extras={"where": "(1=1)"}), # not stored
|
||||
]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, stored_qc)
|
||||
|
||||
|
||||
def test_sql_filters_different_sql_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Modified SQL (appending extra predicates) is blocked."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "col > 5",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
# Attacker appends extra predicate
|
||||
query = QueryObject(
|
||||
extras={"where": "(col > 5) AND (1=1)"},
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_simple_filters_not_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""SIMPLE structured filters (from dashboard native filters) are not blocked."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(
|
||||
filters=[{"col": "country", "op": "==", "val": "US"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
simple_adhoc_filter = {
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "country",
|
||||
"operator": "==",
|
||||
"comparator": "US",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
form_data: dict[str, Any] = {
|
||||
"slice_id": 1,
|
||||
"adhoc_filters": [simple_adhoc_filter],
|
||||
}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_structured_filter_adhoc_col_blocked(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Structured filter with an adhoc SQL column in ``col`` is blocked.
|
||||
|
||||
``ChartDataFilterSchema.col`` is ``fields.Raw``, so an attacker can pass
|
||||
an adhoc column dict that reaches ``adhoc_column_to_sqla`` and executes
|
||||
arbitrary SQL in the WHERE clause.
|
||||
"""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
adhoc_col: Any = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "1; DROP TABLE users--",
|
||||
"label": "x",
|
||||
}
|
||||
query = QueryObject(
|
||||
filters=[{"col": adhoc_col, "op": "!=", "val": "z"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_structured_filter_stored_adhoc_col_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Cross-filter with an adhoc SQL column matching a stored chart dimension
|
||||
is allowed."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {
|
||||
"columns": [
|
||||
{"sqlExpression": "YEAR(order_date)", "label": "order_year"},
|
||||
],
|
||||
}
|
||||
|
||||
adhoc_col: Any = {
|
||||
"sqlExpression": "YEAR(order_date)",
|
||||
"label": "order_year",
|
||||
}
|
||||
query = QueryObject(
|
||||
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_cross_filter_adhoc_col_from_sibling_chart_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Cross-filter with an adhoc SQL column from a sibling chart on the same
|
||||
dashboard is allowed."""
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
# Target chart (chart B) has no custom SQL columns.
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.id = 2
|
||||
stored_chart.params_dict = {"metrics": ["count"]}
|
||||
|
||||
# Source chart (chart A) has the custom SQL dimension.
|
||||
sibling_chart = mocker.MagicMock()
|
||||
sibling_chart.id = 1
|
||||
sibling_chart.params_dict = {
|
||||
"columns": [
|
||||
{"sqlExpression": "YEAR(order_date)", "label": "order_year"},
|
||||
],
|
||||
}
|
||||
|
||||
# Dashboard contains both charts.
|
||||
dashboard = mocker.MagicMock(spec=Dashboard)
|
||||
dashboard.slices = [sibling_chart, stored_chart]
|
||||
|
||||
mocker.patch("superset.db.session.query")
|
||||
db_query = mocker.patch("superset.db.session.query").return_value
|
||||
db_query.filter.return_value.one_or_none.return_value = dashboard
|
||||
mocker.patch(
|
||||
"superset.security_manager.has_guest_access",
|
||||
return_value=True,
|
||||
)
|
||||
|
||||
adhoc_col: Any = {
|
||||
"sqlExpression": "YEAR(order_date)",
|
||||
"label": "order_year",
|
||||
}
|
||||
query = QueryObject(
|
||||
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 10}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_cross_filter_rejected_for_unauthorized_dashboard(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Cross-filter lookup must not use a dashboard the guest has no access to."""
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.id = 2
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
sibling_chart = mocker.MagicMock()
|
||||
sibling_chart.id = 1
|
||||
sibling_chart.params_dict = {
|
||||
"columns": [{"sqlExpression": "YEAR(order_date)", "label": "order_year"}],
|
||||
}
|
||||
|
||||
dashboard = mocker.MagicMock(spec=Dashboard)
|
||||
dashboard.slices = [sibling_chart, stored_chart]
|
||||
|
||||
mocker.patch("superset.db.session.query")
|
||||
db_query = mocker.patch("superset.db.session.query").return_value
|
||||
db_query.filter.return_value.one_or_none.return_value = dashboard
|
||||
mocker.patch(
|
||||
"superset.security_manager.has_guest_access",
|
||||
return_value=False,
|
||||
)
|
||||
|
||||
adhoc_col: Any = {"sqlExpression": "YEAR(order_date)", "label": "order_year"}
|
||||
query = QueryObject(
|
||||
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 999}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_cross_filter_rejected_when_chart_not_on_dashboard(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Cross-filter lookup must verify the target chart belongs to the dashboard."""
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.id = 99 # not on the dashboard
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
sibling_chart = mocker.MagicMock()
|
||||
sibling_chart.id = 1
|
||||
sibling_chart.params_dict = {
|
||||
"columns": [{"sqlExpression": "YEAR(order_date)", "label": "order_year"}],
|
||||
}
|
||||
|
||||
dashboard = mocker.MagicMock(spec=Dashboard)
|
||||
dashboard.slices = [sibling_chart] # stored_chart not here
|
||||
|
||||
mocker.patch("superset.db.session.query")
|
||||
db_query = mocker.patch("superset.db.session.query").return_value
|
||||
db_query.filter.return_value.one_or_none.return_value = dashboard
|
||||
mocker.patch(
|
||||
"superset.security_manager.has_guest_access",
|
||||
return_value=True,
|
||||
)
|
||||
|
||||
adhoc_col: Any = {"sqlExpression": "YEAR(order_date)", "label": "order_year"}
|
||||
query = QueryObject(
|
||||
filters=[{"col": adhoc_col, "op": "==", "val": "2024"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
form_data: dict[str, Any] = {"slice_id": 99, "dashboardId": 10}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_sibling_expressions_cannot_inject_where_having(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Sibling chart column expressions must not legitimize novel WHERE/HAVING."""
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.id = 2
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
# Sibling has a column expression that an attacker tries to use as WHERE.
|
||||
sibling_chart = mocker.MagicMock()
|
||||
sibling_chart.id = 1
|
||||
sibling_chart.params_dict = {
|
||||
"columns": [
|
||||
{"sqlExpression": "(SELECT secret FROM users LIMIT 1)", "label": "x"},
|
||||
],
|
||||
}
|
||||
|
||||
dashboard = mocker.MagicMock(spec=Dashboard)
|
||||
dashboard.slices = [sibling_chart, stored_chart]
|
||||
|
||||
mocker.patch("superset.db.session.query")
|
||||
db_query = mocker.patch("superset.db.session.query").return_value
|
||||
db_query.filter.return_value.one_or_none.return_value = dashboard
|
||||
|
||||
# Attacker injects the sibling expression into extras.where.
|
||||
query = QueryObject(
|
||||
extras={"where": "(SELECT secret FROM users LIMIT 1)"},
|
||||
)
|
||||
query_context.queries = [query]
|
||||
form_data: dict[str, Any] = {"slice_id": 2, "dashboardId": 10}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_collect_allowed_sql_includes_scalar_column_params(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Scalar column params like x_axis contribute their sqlExpression."""
|
||||
from superset.security.manager import _collect_allowed_sql
|
||||
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {
|
||||
"x_axis": {"sqlExpression": "DATE_TRUNC('month', ts)", "label": "m"},
|
||||
"groupby": [{"sqlExpression": "UPPER(country)", "label": "c"}],
|
||||
}
|
||||
|
||||
_, col_allowed = _collect_allowed_sql(stored_chart, None)
|
||||
|
||||
assert "DATE_TRUNC('month', ts)" in col_allowed
|
||||
assert "UPPER(country)" in col_allowed
|
||||
|
||||
|
||||
def test_sql_filters_structured_filter_string_col_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Structured filter with a plain string column is allowed."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(
|
||||
filters=[{"col": "status", "op": "==", "val": "active"}],
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_empty_filter_sentinel_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""The ``(1 = 0)`` sentinel from a required-but-empty native filter is allowed."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(extras={"where": "(1 = 0)"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_double_sentinel_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Two required-but-empty filters compose ``(1 = 0) AND (1 = 0)``."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(extras={"where": "(1 = 0) AND (1 = 0)"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_stored_clause_plus_sentinel_allowed(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""A stored SQL filter composed with the empty-filter sentinel is allowed."""
|
||||
sql_filter = {
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "region = 'EMEA'",
|
||||
"clause": "WHERE",
|
||||
}
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {"adhoc_filters": [sql_filter]}
|
||||
|
||||
query = QueryObject(
|
||||
extras={"where": "(region = 'EMEA') AND (1 = 0)"},
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_non_dict_adhoc_filter_skipped(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Non-dict items in adhoc_filters are skipped, not 500."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject()
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {
|
||||
"slice_id": 1,
|
||||
"adhoc_filters": ["not_a_dict", 42, None],
|
||||
}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_raise_for_access_guest_user_sql_filter_injection_blocked(
|
||||
mocker: MockerFixture,
|
||||
app_context: None,
|
||||
stored_metrics: list[AdhocMetric],
|
||||
) -> None:
|
||||
"""Guest user injecting SQL via extras.where is rejected by raise_for_access."""
|
||||
sm = SupersetSecurityManager(appbuilder)
|
||||
mocker.patch.object(sm, "is_guest_user", return_value=True)
|
||||
mocker.patch.object(sm, "can_access", return_value=True)
|
||||
|
||||
query_context = mocker.MagicMock()
|
||||
query_context.slice_.id = 42
|
||||
query_context.slice_.query_context = None
|
||||
query_context.slice_.params_dict = {"metrics": stored_metrics}
|
||||
|
||||
query_context.form_data = {"slice_id": 42, "metrics": stored_metrics}
|
||||
query_context.queries = [
|
||||
QueryObject(
|
||||
metrics=stored_metrics, # type: ignore
|
||||
extras={"where": "1=1 UNION SELECT password FROM users"},
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(SupersetSecurityException):
|
||||
sm.raise_for_access(query_context=query_context)
|
||||
|
||||
|
||||
def test_sql_filters_cache_replay_skips_check(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Cache-replay requests skip the SQL filter check."""
|
||||
query_context = mocker.MagicMock()
|
||||
query_context._from_cache_replay = True
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(extras={"where": "(injected SQL)"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert not _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_column_expression_cannot_become_where(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""A chart's column sqlExpression must not be injectable as extras.where."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {
|
||||
"columns": [
|
||||
{
|
||||
"sqlExpression": "(SELECT secret FROM users LIMIT 1)",
|
||||
"label": "x",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
query = QueryObject(
|
||||
extras={"where": "((SELECT secret FROM users LIMIT 1))"},
|
||||
)
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
|
||||
def test_sql_filters_unbalanced_parens_rejected(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Unbalanced parens in extras.where are rejected (403, not 500)."""
|
||||
query_context = mocker.MagicMock()
|
||||
stored_chart = mocker.MagicMock()
|
||||
stored_chart.params_dict = {}
|
||||
|
||||
query = QueryObject(extras={"where": "(a) AND (b"})
|
||||
query_context.queries = [query]
|
||||
|
||||
form_data: dict[str, Any] = {"slice_id": 1}
|
||||
|
||||
assert _sql_filters_modified(query_context, form_data, stored_chart, None)
|
||||
|
||||
@@ -38,21 +38,6 @@ def test_column_with_valid_operation():
|
||||
assert (get_metric_type_from_column(column, datasource)) == "floating"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression",
|
||||
[
|
||||
"stddev_samp(my_column)",
|
||||
"STDDEV_SAMP (my_column)",
|
||||
"StdDev_Samp(my_column)",
|
||||
],
|
||||
)
|
||||
def test_column_with_lowercase_or_whitespaced_operation(expression):
|
||||
metric = SqlMetric(metric_name="my_column", expression=expression)
|
||||
datasource = MagicMock(metrics=[metric])
|
||||
column = "my_column"
|
||||
assert (get_metric_type_from_column(column, datasource)) == "floating"
|
||||
|
||||
|
||||
def test_column_with_invalid_operation():
|
||||
metric = SqlMetric(metric_name="my_column", expression="INVALID(my_column)")
|
||||
datasource = MagicMock(metrics=[metric])
|
||||
|
||||
Reference in New Issue
Block a user