mirror of
https://github.com/apache/superset.git
synced 2026-07-25 16:12:39 +00:00
Compare commits
7 Commits
dashboard-
...
hughhhh/ha
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
670a95215c | ||
|
|
719649fe1a | ||
|
|
de7dfc1585 | ||
|
|
a1212ec089 | ||
|
|
2bbada837b | ||
|
|
de29e984ff | ||
|
|
c7a6c6fc9c |
13
UPDATING.md
13
UPDATING.md
@@ -32,12 +32,21 @@ A new dashboard action exports every chart's data to a single multi-sheet
|
||||
requires a running Celery worker and a configured SMTP transport, since the task
|
||||
emails the requesting user a pre-signed download link. New config keys:
|
||||
`EXCEL_EXPORT_S3_BUCKET`, `EXCEL_EXPORT_S3_KEY_PREFIX`,
|
||||
`EXCEL_EXPORT_LINK_TTL_SECONDS`, `EXCEL_EXPORT_S3_CLIENT_KWARGS`, and
|
||||
`EXCEL_EXPORT_TABLE_VIZ_TYPES`.
|
||||
`EXCEL_EXPORT_LINK_TTL_SECONDS`, `EXCEL_EXPORT_S3_CLIENT_KWARGS`,
|
||||
`EXCEL_EXPORT_TABLE_VIZ_TYPES`, and `EXCEL_EXPORT_REBUILD_VIZ_TYPES`.
|
||||
|
||||
The feature depends on `boto3`, which is **not** installed by default; install it
|
||||
with `pip install apache-superset[excel-export]`.
|
||||
|
||||
Charts store their `query_context` only once they have been (re-)saved in
|
||||
Explore, so older charts may have none. For a conservative allowlist of viz types
|
||||
(`EXCEL_EXPORT_REBUILD_VIZ_TYPES`, default `table`, `big_number_total`,
|
||||
`big_number`, `pie`) the export rebuilds a query context from the chart's saved
|
||||
form data so those charts still export. The rebuild is a generic single-query
|
||||
mapping and does **not** reproduce plugin post-processing (pivot, rolling,
|
||||
forecast) or multi-query charts, so any chart of another type without a saved
|
||||
query context is skipped and listed in the email for the user to re-save.
|
||||
|
||||
A second mode, **Export Images to Excel**, embeds non-table charts as rendered
|
||||
images (which viz types stay tabular is controlled by
|
||||
`EXCEL_EXPORT_TABLE_VIZ_TYPES`). It renders through the headless webdriver, so the
|
||||
|
||||
227
superset/common/form_data_query_context.py
Normal file
227
superset/common/form_data_query_context.py
Normal file
@@ -0,0 +1,227 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Synthesize a query context from a chart's saved form data (``params``).
|
||||
|
||||
A chart's ``query_context`` is normally generated client-side by each viz
|
||||
plugin's ``buildQuery`` and only persisted when the chart is (re-)saved in
|
||||
Explore. Charts that predate that behavior keep their ``params`` (form data) but
|
||||
carry no ``query_context``, so server-side consumers that need to run the query
|
||||
(e.g. the dashboard Excel export) have nothing to execute.
|
||||
|
||||
This module rebuilds a best-effort query context from the form data — columns,
|
||||
metrics, filters (including free-form SQL and the time range), ordering and time
|
||||
grain — mirroring the shared parts of the viz plugins' ``buildQuery``. It does
|
||||
**not** reproduce plugin post-processing (pivot, contribution/percent
|
||||
transforms, rolling/forecast) or multi-query fan-out, so callers must restrict it
|
||||
to viz types whose data maps faithfully to a single plain query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from superset.utils import json
|
||||
|
||||
|
||||
def adhoc_filters_to_query_filters(
|
||||
adhoc_filters: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Convert ``SIMPLE`` adhoc filters into QueryObject filter clauses.
|
||||
|
||||
Adhoc filters use ``{subject, operator, comparator}`` while a query object
|
||||
expects ``{col, op, val}``. Only ``SIMPLE`` WHERE-clause filters are
|
||||
convertible here; free-form ``SQL`` filters have no ``{col, op, val}``
|
||||
equivalent and are handled separately (see :func:`freeform_where_having`).
|
||||
"""
|
||||
result: list[dict[str, Any]] = []
|
||||
for flt in adhoc_filters or []:
|
||||
if (
|
||||
flt.get("expressionType") == "SIMPLE"
|
||||
and (flt.get("clause") or "WHERE").upper() == "WHERE"
|
||||
):
|
||||
result.append(
|
||||
{
|
||||
"col": flt.get("subject"),
|
||||
"op": flt.get("operator"),
|
||||
"val": flt.get("comparator"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def freeform_where_having(form_data: dict[str, Any]) -> dict[str, str]:
|
||||
"""
|
||||
Collect free-form SQL predicates into a query ``extras`` mapping.
|
||||
|
||||
Mirrors ``processFilters`` on the frontend: ``SQL`` adhoc filters (and a
|
||||
legacy top-level ``where``) join into ``extras.where`` / ``extras.having`` by
|
||||
clause, so a chart restricted by a custom SQL predicate exports the same rows
|
||||
it displays instead of the full, unrestricted result.
|
||||
"""
|
||||
where: list[str] = []
|
||||
having: list[str] = []
|
||||
if form_data.get("where"):
|
||||
where.append(form_data["where"])
|
||||
for flt in form_data.get("adhoc_filters") or []:
|
||||
if flt.get("expressionType") == "SQL" and flt.get("sqlExpression"):
|
||||
clause = (flt.get("clause") or "WHERE").upper()
|
||||
(having if clause == "HAVING" else where).append(flt["sqlExpression"])
|
||||
|
||||
extras: dict[str, str] = {}
|
||||
if where:
|
||||
extras["where"] = " AND ".join(f"({clause})" for clause in where)
|
||||
if having:
|
||||
extras["having"] = " AND ".join(f"({clause})" for clause in having)
|
||||
return extras
|
||||
|
||||
|
||||
def columns_from_form_data(form_data: dict[str, Any]) -> list[Any]:
|
||||
"""
|
||||
Derive the query's grouping/raw columns from form data.
|
||||
|
||||
Handles raw-mode tables (``all_columns``/``columns``), an ``x_axis`` (string
|
||||
or adhoc column), and ``groupby`` dimensions, de-duplicating while preserving
|
||||
order.
|
||||
"""
|
||||
if form_data.get("query_mode") == "raw" and (
|
||||
form_data.get("all_columns") or form_data.get("columns")
|
||||
):
|
||||
return list(form_data.get("all_columns") or form_data.get("columns") or [])
|
||||
|
||||
groupby_columns: list[Any] = form_data.get("groupby") or []
|
||||
raw_columns: list[Any] = form_data.get("columns") or []
|
||||
# Prefer explicit raw columns only when they are actually present; a stale
|
||||
# empty ``columns: []`` key must not shadow the group-by dimensions (which
|
||||
# would silently drop the grouping and change the aggregation).
|
||||
columns = raw_columns.copy() if raw_columns else groupby_columns.copy()
|
||||
|
||||
x_axis = form_data.get("x_axis")
|
||||
if isinstance(x_axis, str) and x_axis and x_axis not in columns:
|
||||
columns.insert(0, x_axis)
|
||||
elif isinstance(x_axis, dict):
|
||||
col_name = x_axis.get("column_name")
|
||||
if col_name and col_name not in columns:
|
||||
columns.insert(0, col_name)
|
||||
return columns
|
||||
|
||||
|
||||
def orderby_from_form_data(
|
||||
form_data: dict[str, Any], metrics: list[Any]
|
||||
) -> list[list[Any]]:
|
||||
"""
|
||||
Derive ordering so a ``row_limit`` returns the chart's top-N, not an
|
||||
arbitrary N.
|
||||
|
||||
Raw-mode tables order by ``order_by_cols`` (stored as JSON ``[col, asc]``
|
||||
pairs). Aggregate charts order by the configured sort metric
|
||||
(``timeseries_limit_metric``, or the first metric when ``sort_by_metric`` is
|
||||
set), otherwise fall back to the first metric descending — matching the
|
||||
table/pie ``buildQuery`` defaults.
|
||||
"""
|
||||
if order_by_cols := form_data.get("order_by_cols") or []:
|
||||
parsed: list[list[Any]] = []
|
||||
for col in order_by_cols:
|
||||
if isinstance(col, str):
|
||||
try:
|
||||
col = json.loads(col)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
parsed.append(col)
|
||||
return parsed
|
||||
|
||||
if not metrics:
|
||||
return []
|
||||
|
||||
order_desc = form_data.get("order_desc", True)
|
||||
sort_metric = form_data.get("timeseries_limit_metric") or (
|
||||
metrics[0] if form_data.get("sort_by_metric") else None
|
||||
)
|
||||
if sort_metric is not None:
|
||||
return [[sort_metric, not order_desc]]
|
||||
# No explicit sort metric: default to the first metric, descending.
|
||||
return [[metrics[0], False]]
|
||||
|
||||
|
||||
def build_query_context_from_form_data(
|
||||
form_data: dict[str, Any],
|
||||
datasource: dict[str, Any],
|
||||
viz_type: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Build a query-context payload (the JSON shape ``ChartDataQueryContextSchema``
|
||||
loads) from a chart's form data and datasource reference.
|
||||
|
||||
:param form_data: The chart's saved ``params`` parsed to a dict.
|
||||
:param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
|
||||
:param viz_type: The chart's viz type, used for viz-specific handling.
|
||||
:returns: A single-query query-context dict.
|
||||
"""
|
||||
metrics = list(form_data.get("metrics") or [])
|
||||
# Single-metric charts (e.g. Big Number) store ``metric`` rather than
|
||||
# ``metrics``.
|
||||
if not metrics and form_data.get("metric"):
|
||||
metrics = [form_data["metric"]]
|
||||
# Table percent metrics are computed as additional query metrics.
|
||||
if viz_type == "table" and form_data.get("percent_metrics"):
|
||||
metrics = [*metrics, *form_data["percent_metrics"]]
|
||||
|
||||
columns = columns_from_form_data(form_data)
|
||||
# Only a Big Number *with a trendline* (viz_type ``big_number``) groups by its
|
||||
# time column; ``big_number_total`` is a single aggregate and must not be
|
||||
# grouped, or it would return one row per timestamp instead of a total.
|
||||
if not columns and viz_type == "big_number" and form_data.get("granularity_sqla"):
|
||||
columns = [form_data["granularity_sqla"]]
|
||||
|
||||
# SIMPLE adhoc filters (+ legacy top-level ``filters``) become query filters;
|
||||
# free-form SQL predicates go into ``extras``.
|
||||
filters = adhoc_filters_to_query_filters(form_data.get("adhoc_filters", []))
|
||||
for flt in form_data.get("filters") or []:
|
||||
if isinstance(flt, dict) and flt.get("col") is not None:
|
||||
filters.append(flt)
|
||||
|
||||
extras = freeform_where_having(form_data)
|
||||
if form_data.get("time_grain_sqla"):
|
||||
extras["time_grain_sqla"] = form_data["time_grain_sqla"]
|
||||
|
||||
time_range = form_data.get("time_range") or "No filter"
|
||||
query: dict[str, Any] = {
|
||||
"columns": columns,
|
||||
"metrics": metrics,
|
||||
"orderby": orderby_from_form_data(form_data, metrics),
|
||||
"filters": filters,
|
||||
"time_range": time_range,
|
||||
}
|
||||
if extras:
|
||||
query["extras"] = extras
|
||||
# ``granularity`` names the temporal column that actually applies the time
|
||||
# range; without it, ``time_range`` is inert and the export returns the entire
|
||||
# history. Only set it when there is a real range to apply — otherwise a
|
||||
# numeric column saved as ``granularity_sqla`` (with no active range) would be
|
||||
# forced through date bucketing and fail.
|
||||
granularity = form_data.get("granularity") or form_data.get("granularity_sqla")
|
||||
if granularity and time_range != "No filter":
|
||||
query["granularity"] = granularity
|
||||
if form_data.get("row_limit"):
|
||||
query["row_limit"] = form_data["row_limit"]
|
||||
|
||||
return {
|
||||
"datasource": datasource,
|
||||
"queries": [query],
|
||||
"form_data": form_data,
|
||||
}
|
||||
@@ -1464,6 +1464,16 @@ EXCEL_EXPORT_S3_CLIENT_KWARGS: dict[str, Any] = {}
|
||||
# a rendered image. Set to None to fall back to the built-in default.
|
||||
EXCEL_EXPORT_TABLE_VIZ_TYPES: set[str] | None = None
|
||||
|
||||
# Viz types whose data export may synthesize a query context from the chart's
|
||||
# saved form data when no ``query_context`` was persisted (older charts store
|
||||
# ``params`` but not a query context). Restricted to viz types whose data maps
|
||||
# faithfully to a single plain query; charts of any other type without a saved
|
||||
# query context are skipped and listed for the user to re-save in Explore. The
|
||||
# rebuild does not reproduce plugin post-processing (pivot, rolling, forecast) or
|
||||
# multi-query charts, so keep this conservative. Set to None to fall back to the
|
||||
# built-in default.
|
||||
EXCEL_EXPORT_REBUILD_VIZ_TYPES: set[str] | None = None
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Time grain configurations
|
||||
# ---------------------------------------------------
|
||||
|
||||
@@ -441,18 +441,14 @@ def adhoc_filters_to_query_filters(
|
||||
|
||||
Adhoc filters use ``{subject, operator, comparator}`` keys while
|
||||
``QueryContextFactory`` expects ``{col, op, val}`` (QueryObjectFilterClause).
|
||||
Delegates to the shared builder so the MCP and dashboard-export paths stay in
|
||||
sync (single source of truth).
|
||||
"""
|
||||
result: list[Dict[str, Any]] = []
|
||||
for f in adhoc_filters:
|
||||
if f.get("expressionType") == "SIMPLE":
|
||||
result.append(
|
||||
{
|
||||
"col": f.get("subject"),
|
||||
"op": f.get("operator"),
|
||||
"val": f.get("comparator"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
from superset.common.form_data_query_context import (
|
||||
adhoc_filters_to_query_filters as _shared,
|
||||
)
|
||||
|
||||
return _shared(adhoc_filters)
|
||||
|
||||
|
||||
def map_table_config(config: TableChartConfig) -> Dict[str, Any]:
|
||||
|
||||
@@ -39,26 +39,14 @@ SUPPORTED_FORM_DATA_PREVIEW_FORMATS = frozenset({"ascii", "table", "vega_lite"})
|
||||
|
||||
|
||||
def _build_query_columns(form_data: Dict[str, Any]) -> list[str]:
|
||||
"""Build query columns list from form_data, including both x_axis and groupby."""
|
||||
# Table charts in raw mode use all_columns or columns
|
||||
all_columns = form_data.get("all_columns", [])
|
||||
raw_columns_field = form_data.get("columns", [])
|
||||
if form_data.get("query_mode") == "raw" and (all_columns or raw_columns_field):
|
||||
return list(all_columns or raw_columns_field)
|
||||
"""Build query columns list from form_data, including both x_axis and groupby.
|
||||
|
||||
x_axis_config = form_data.get("x_axis")
|
||||
groupby_columns: list[str] = form_data.get("groupby") or []
|
||||
raw_columns: list[str] = form_data.get("columns") or []
|
||||
Delegates to the shared builder so the MCP and dashboard-export paths stay in
|
||||
sync (single source of truth).
|
||||
"""
|
||||
from superset.common.form_data_query_context import columns_from_form_data
|
||||
|
||||
columns = raw_columns.copy() if "columns" in form_data else groupby_columns.copy()
|
||||
if x_axis_config and isinstance(x_axis_config, str):
|
||||
if x_axis_config not in columns:
|
||||
columns.insert(0, x_axis_config)
|
||||
elif x_axis_config and isinstance(x_axis_config, dict):
|
||||
col_name = x_axis_config.get("column_name")
|
||||
if col_name and col_name not in columns:
|
||||
columns.insert(0, col_name)
|
||||
return columns
|
||||
return columns_from_form_data(form_data)
|
||||
|
||||
|
||||
def generate_preview_from_form_data(
|
||||
|
||||
@@ -47,6 +47,7 @@ from superset.charts.schemas import ChartDataQueryContextSchema
|
||||
from superset.commands.chart.data.get_data_command import ChartDataCommand
|
||||
from superset.commands.distributed_lock.release import ReleaseDistributedLock
|
||||
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
|
||||
from superset.common.form_data_query_context import build_query_context_from_form_data
|
||||
from superset.dashboards.excel_export import email
|
||||
from superset.dashboards.excel_export.layout import get_charts_in_layout_order
|
||||
from superset.dashboards.excel_export.screenshot import render_chart_image
|
||||
@@ -67,6 +68,12 @@ EXPORT_MODE_IMAGES = "images"
|
||||
# image. Operators can override the set via ``EXCEL_EXPORT_TABLE_VIZ_TYPES``.
|
||||
TABLE_VIZ_TYPES = {"table", "pivot_table_v2", "pivot_table"}
|
||||
|
||||
# Viz types whose missing query context may be rebuilt from saved form data.
|
||||
# Conservative by default: only charts whose data maps faithfully to a single
|
||||
# plain query (no post-processing, no multi-query fan-out). Operators can
|
||||
# override via ``EXCEL_EXPORT_REBUILD_VIZ_TYPES``.
|
||||
REBUILD_VIZ_TYPES = {"table", "big_number_total", "big_number", "pie"}
|
||||
|
||||
EXPORT_SOFT_TIME_LIMIT = 600
|
||||
EXPORT_HARD_TIME_LIMIT = 660
|
||||
|
||||
@@ -96,6 +103,67 @@ def _chart_label(chart: Any) -> str:
|
||||
return f"{chart.id} - {chart.slice_name or ''}".strip()
|
||||
|
||||
|
||||
def _saved_query_context(raw: Any) -> dict[str, Any] | None:
|
||||
"""
|
||||
The chart's saved query context parsed to a dict, or ``None`` when it is
|
||||
missing or unusable.
|
||||
|
||||
Returns ``None`` for a blank value, a string that does not parse as JSON, a
|
||||
value that parses to something other than an object (e.g. ``"null"``), and an
|
||||
object with no queries (e.g. ``"{}"`` or ``{"queries": []}``) — all treated
|
||||
the same as a missing context.
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not isinstance(parsed, dict) or not parsed.get("queries"):
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def _rebuild_viz_types() -> set[str]:
|
||||
"""Viz types eligible for form-data query-context rebuild (config or default).
|
||||
|
||||
Only ``None`` falls back to the default; an explicitly configured empty set is
|
||||
honored so operators can disable the rebuild entirely.
|
||||
"""
|
||||
configured = current_app.config.get("EXCEL_EXPORT_REBUILD_VIZ_TYPES")
|
||||
return REBUILD_VIZ_TYPES if configured is None else configured
|
||||
|
||||
|
||||
def _resolve_query_context(chart: Any) -> dict[str, Any] | None:
|
||||
"""
|
||||
The query-context payload to run for a chart's data export, or ``None`` when
|
||||
none can be obtained.
|
||||
|
||||
Prefers the chart's saved ``query_context``. When that is missing or empty,
|
||||
synthesizes one from the chart's saved form data (``params``) — but only for
|
||||
viz types whose data maps faithfully to a single plain query
|
||||
(``EXCEL_EXPORT_REBUILD_VIZ_TYPES``); other charts return ``None`` so the
|
||||
caller lists them for re-saving rather than exporting inaccurate data.
|
||||
"""
|
||||
if saved := _saved_query_context(chart.query_context):
|
||||
return saved
|
||||
|
||||
if chart.viz_type not in _rebuild_viz_types() or chart.datasource_id is None:
|
||||
return None
|
||||
try:
|
||||
form_data = json.loads(chart.params) if chart.params else {}
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not isinstance(form_data, dict) or not form_data:
|
||||
return None
|
||||
|
||||
return build_query_context_from_form_data(
|
||||
form_data,
|
||||
{"id": chart.datasource_id, "type": chart.datasource_type or "table"},
|
||||
chart.viz_type,
|
||||
)
|
||||
|
||||
|
||||
def _record_to_row(record: dict[str, Any], colnames: list[str]) -> list[Any]:
|
||||
return [record.get(col) for col in colnames]
|
||||
|
||||
@@ -131,17 +199,24 @@ def _write_chart_image_sheet(
|
||||
def _write_chart_sheets(
|
||||
writer: StreamingXlsxWriter,
|
||||
chart: Any,
|
||||
json_body: dict[str, Any],
|
||||
dashboard_id: int,
|
||||
active_data_mask: dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Run a single chart's query and stream its result(s) into the workbook.
|
||||
|
||||
Charts may yield more than one query (e.g. mixed-series charts); each becomes
|
||||
its own sheet. Raises if the chart cannot be exported, so the caller can skip
|
||||
it and note it in the email.
|
||||
``json_body`` is the resolved query-context payload (the chart's saved
|
||||
context or one synthesized from its form data). Charts may yield more than
|
||||
one query (e.g. mixed-series charts); each becomes its own sheet. Raises if
|
||||
the chart cannot be exported, so the caller can skip it and note it in the
|
||||
email.
|
||||
"""
|
||||
json_body = json.loads(chart.query_context)
|
||||
# Shallow-copy before setting our own top-level keys so the caller's payload
|
||||
# keeps its original result_format/result_type. (The nested ``queries`` are
|
||||
# mutated in place by apply_dashboard_filter_context below, which is safe here
|
||||
# because each body is freshly built or parsed per chart.)
|
||||
json_body = dict(json_body)
|
||||
# Override any stale saved values: we always want full JSON results.
|
||||
json_body["result_format"] = ChartDataResultFormat.JSON
|
||||
json_body["result_type"] = ChartDataResultType.FULL
|
||||
@@ -196,19 +271,26 @@ def _build_workbook(
|
||||
try:
|
||||
for chart in get_charts_in_layout_order(dashboard):
|
||||
label = _chart_label(chart)
|
||||
as_image = _renders_as_image(chart, mode)
|
||||
# Image charts render from their saved params and don't need a query
|
||||
# context; data (and table) charts still do.
|
||||
if not as_image and not chart.query_context:
|
||||
errored.setdefault(email.ERROR_NO_QUERY_CONTEXT, []).append(label)
|
||||
continue
|
||||
try:
|
||||
if as_image:
|
||||
if _renders_as_image(chart, mode):
|
||||
# Image charts render from their saved params via the
|
||||
# webdriver and don't need a query context.
|
||||
_write_chart_image_sheet(
|
||||
writer, chart, dashboard.id, active_data_mask, user
|
||||
)
|
||||
else:
|
||||
_write_chart_sheets(writer, chart, dashboard.id, active_data_mask)
|
||||
# Data charts need a query context: use the saved one, or
|
||||
# rebuild it from form data for eligible viz types. Skip
|
||||
# cleanly when none is available rather than failing.
|
||||
json_body = _resolve_query_context(chart)
|
||||
if json_body is None:
|
||||
errored.setdefault(email.ERROR_NO_QUERY_CONTEXT, []).append(
|
||||
label
|
||||
)
|
||||
continue
|
||||
_write_chart_sheets(
|
||||
writer, chart, json_body, dashboard.id, active_data_mask
|
||||
)
|
||||
except SoftTimeLimitExceeded:
|
||||
# A soft timeout is a task-level signal, not a per-chart failure:
|
||||
# let it propagate so the outer handler emails a failure and runs
|
||||
|
||||
268
tests/unit_tests/common/test_form_data_query_context.py
Normal file
268
tests/unit_tests/common/test_form_data_query_context.py
Normal file
@@ -0,0 +1,268 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from superset.common.form_data_query_context import (
|
||||
adhoc_filters_to_query_filters,
|
||||
build_query_context_from_form_data,
|
||||
columns_from_form_data,
|
||||
)
|
||||
|
||||
DATASOURCE = {"id": 7, "type": "table"}
|
||||
|
||||
|
||||
def test_adhoc_filters_converts_simple_and_drops_custom_sql() -> None:
|
||||
adhoc = [
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "country",
|
||||
"operator": "==",
|
||||
"comparator": "US",
|
||||
},
|
||||
{"expressionType": "SQL", "sqlExpression": "1 = 1"},
|
||||
]
|
||||
assert adhoc_filters_to_query_filters(adhoc) == [
|
||||
{"col": "country", "op": "==", "val": "US"}
|
||||
]
|
||||
assert adhoc_filters_to_query_filters([]) == []
|
||||
|
||||
|
||||
def test_columns_prefers_groupby_and_x_axis() -> None:
|
||||
form_data = {"groupby": ["region"], "x_axis": "ds"}
|
||||
assert columns_from_form_data(form_data) == ["ds", "region"]
|
||||
|
||||
|
||||
def test_columns_raw_mode_uses_all_columns() -> None:
|
||||
form_data = {"query_mode": "raw", "all_columns": ["a", "b"]}
|
||||
assert columns_from_form_data(form_data) == ["a", "b"]
|
||||
|
||||
|
||||
def test_columns_x_axis_as_adhoc_dict() -> None:
|
||||
# An x_axis stored as an adhoc column dict contributes its column_name,
|
||||
# prepended ahead of the groupby dimensions.
|
||||
form_data = {"groupby": ["region"], "x_axis": {"column_name": "ds"}}
|
||||
assert columns_from_form_data(form_data) == ["ds", "region"]
|
||||
|
||||
|
||||
def test_columns_x_axis_dict_without_column_name_is_ignored() -> None:
|
||||
form_data = {
|
||||
"groupby": ["region"],
|
||||
"x_axis": {"label": "custom", "sqlExpression": "a+b"},
|
||||
}
|
||||
assert columns_from_form_data(form_data) == ["region"]
|
||||
|
||||
|
||||
def test_columns_empty_columns_key_does_not_shadow_groupby() -> None:
|
||||
# A stale, explicitly-present-but-empty ``columns`` key must not drop the
|
||||
# group-by dimensions (which would silently change the aggregation).
|
||||
form_data = {"groupby": ["country"], "columns": []}
|
||||
assert columns_from_form_data(form_data) == ["country"]
|
||||
|
||||
|
||||
def test_build_context_maps_groupby_metrics_and_filters() -> None:
|
||||
form_data = {
|
||||
"groupby": ["country"],
|
||||
"metrics": ["count"],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "year",
|
||||
"operator": ">",
|
||||
"comparator": 2000,
|
||||
},
|
||||
],
|
||||
"time_range": "Last year",
|
||||
"row_limit": 500,
|
||||
}
|
||||
|
||||
ctx = build_query_context_from_form_data(form_data, DATASOURCE)
|
||||
|
||||
assert ctx["datasource"] == DATASOURCE
|
||||
assert ctx["form_data"] == form_data
|
||||
assert len(ctx["queries"]) == 1
|
||||
query = ctx["queries"][0]
|
||||
assert query["columns"] == ["country"]
|
||||
assert query["metrics"] == ["count"]
|
||||
assert query["filters"] == [{"col": "year", "op": ">", "val": 2000}]
|
||||
assert query["time_range"] == "Last year"
|
||||
assert query["row_limit"] == 500
|
||||
|
||||
|
||||
def test_build_context_big_number_singular_metric_and_default_time_range() -> None:
|
||||
form_data = {"metric": "sum__sales"}
|
||||
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
|
||||
assert query["metrics"] == ["sum__sales"]
|
||||
assert query["time_range"] == "No filter"
|
||||
# No row_limit in form data → not forced into the query.
|
||||
assert "row_limit" not in query
|
||||
|
||||
|
||||
def test_build_context_merges_legacy_and_adhoc_filters() -> None:
|
||||
# Legacy charts store simple filters directly under ``filters`` (already in
|
||||
# QueryObject shape); they are honored alongside adhoc_filters, and malformed
|
||||
# entries are dropped.
|
||||
form_data = {
|
||||
"groupby": ["country"],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "year",
|
||||
"operator": ">",
|
||||
"comparator": 2000,
|
||||
},
|
||||
],
|
||||
"filters": [
|
||||
{"col": "region", "op": "==", "val": "EMEA"},
|
||||
{"not_a_filter": True},
|
||||
],
|
||||
}
|
||||
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
|
||||
assert query["filters"] == [
|
||||
{"col": "year", "op": ">", "val": 2000},
|
||||
{"col": "region", "op": "==", "val": "EMEA"},
|
||||
]
|
||||
|
||||
|
||||
def test_big_number_trendline_promotes_granularity_sqla_column() -> None:
|
||||
# A Big Number *with a trendline* (viz_type "big_number") has no
|
||||
# groupby/columns; its time column (granularity_sqla) becomes the sole column.
|
||||
form_data = {"metric": "count", "granularity_sqla": "order_date"}
|
||||
|
||||
query = build_query_context_from_form_data(
|
||||
form_data, DATASOURCE, viz_type="big_number"
|
||||
)["queries"][0]
|
||||
|
||||
assert query["columns"] == ["order_date"]
|
||||
assert query["metrics"] == ["count"]
|
||||
|
||||
|
||||
def test_big_number_total_does_not_promote_granularity_sqla_column() -> None:
|
||||
# big_number_total is a single aggregate; promoting granularity_sqla to a
|
||||
# column would turn one total into one row per timestamp.
|
||||
form_data = {"metric": "count", "granularity_sqla": "order_date"}
|
||||
|
||||
query = build_query_context_from_form_data(
|
||||
form_data, DATASOURCE, viz_type="big_number_total"
|
||||
)["queries"][0]
|
||||
|
||||
assert query["columns"] == []
|
||||
|
||||
|
||||
def test_build_context_sets_granularity_for_time_filtering() -> None:
|
||||
# Without a `granularity`, the `time_range` is inert downstream, so a legacy
|
||||
# chart with granularity_sqla + time_range would export its full history.
|
||||
form_data = {
|
||||
"metrics": ["count"],
|
||||
"granularity_sqla": "ds",
|
||||
"time_range": "Last quarter",
|
||||
}
|
||||
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
|
||||
assert query["granularity"] == "ds"
|
||||
assert query["time_range"] == "Last quarter"
|
||||
|
||||
|
||||
def test_build_context_prefers_explicit_granularity_over_sqla() -> None:
|
||||
form_data = {
|
||||
"metrics": ["count"],
|
||||
"granularity": "event_time",
|
||||
"granularity_sqla": "ds",
|
||||
"time_range": "Last week",
|
||||
}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert query["granularity"] == "event_time"
|
||||
|
||||
|
||||
def test_build_context_omits_granularity_without_active_time_range() -> None:
|
||||
# A numeric column saved as granularity_sqla with no active range must not be
|
||||
# forced through temporal bucketing (which would fail on non-temporal columns).
|
||||
form_data = {"metrics": ["count"], "granularity_sqla": "year"}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert "granularity" not in query
|
||||
|
||||
|
||||
def test_orderby_defaults_to_first_metric_descending() -> None:
|
||||
# With a row_limit, ordering must be deterministic so the export returns the
|
||||
# chart's top-N, not an arbitrary N.
|
||||
form_data = {"metrics": ["count"], "groupby": ["c"], "row_limit": 10}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert query["orderby"] == [["count", False]]
|
||||
|
||||
|
||||
def test_orderby_uses_timeseries_limit_metric_and_order_desc() -> None:
|
||||
form_data = {
|
||||
"metrics": ["count"],
|
||||
"groupby": ["c"],
|
||||
"timeseries_limit_metric": "revenue",
|
||||
"order_desc": False,
|
||||
}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert query["orderby"] == [["revenue", True]]
|
||||
|
||||
|
||||
def test_orderby_pie_sort_by_metric() -> None:
|
||||
form_data = {"metric": "count", "groupby": ["c"], "sort_by_metric": True}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE, viz_type="pie")[
|
||||
"queries"
|
||||
][0]
|
||||
assert query["orderby"] == [["count", False]]
|
||||
|
||||
|
||||
def test_orderby_raw_mode_parses_order_by_cols() -> None:
|
||||
form_data = {
|
||||
"query_mode": "raw",
|
||||
"all_columns": ["a"],
|
||||
# A malformed entry is skipped rather than raising.
|
||||
"order_by_cols": ['["a", true]', "not json", ["b", False]],
|
||||
}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert query["orderby"] == [["a", True], ["b", False]]
|
||||
|
||||
|
||||
def test_sql_filters_and_legacy_where_go_into_extras() -> None:
|
||||
form_data = {
|
||||
"groupby": ["c"],
|
||||
"where": "region = 'EMEA'",
|
||||
"adhoc_filters": [
|
||||
{"expressionType": "SQL", "clause": "WHERE", "sqlExpression": "sales > 0"},
|
||||
{
|
||||
"expressionType": "SQL",
|
||||
"clause": "HAVING",
|
||||
"sqlExpression": "SUM(x) > 5",
|
||||
},
|
||||
],
|
||||
}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE)["queries"][0]
|
||||
assert query["extras"]["where"] == "(region = 'EMEA') AND (sales > 0)"
|
||||
assert query["extras"]["having"] == "(SUM(x) > 5)"
|
||||
|
||||
|
||||
def test_table_percent_metrics_and_time_grain_are_carried() -> None:
|
||||
form_data = {
|
||||
"groupby": ["c"],
|
||||
"metrics": ["count"],
|
||||
"percent_metrics": ["pct_total"],
|
||||
"time_grain_sqla": "P1M",
|
||||
}
|
||||
query = build_query_context_from_form_data(form_data, DATASOURCE, viz_type="table")[
|
||||
"queries"
|
||||
][0]
|
||||
assert query["metrics"] == ["count", "pct_total"]
|
||||
assert query["extras"]["time_grain_sqla"] == "P1M"
|
||||
@@ -182,6 +182,184 @@ def test_chart_without_query_context_is_skipped(mocks: dict[str, Any]) -> None:
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_context",
|
||||
[
|
||||
"", # blank
|
||||
"null", # parses to None
|
||||
"{}", # object with no queries
|
||||
'{"queries": []}', # object with an empty queries list
|
||||
"not valid json", # unparseable
|
||||
],
|
||||
)
|
||||
def test_chart_with_empty_query_context_is_skipped(
|
||||
mocks: dict[str, Any], raw_context: str
|
||||
) -> None:
|
||||
# A present-but-empty/unusable query context is treated the same as a
|
||||
# missing one: the chart is listed under "no query context" and the export
|
||||
# continues, rather than raising mid-export and landing in the general bucket.
|
||||
good = _chart(10, "Good")
|
||||
empty = _chart(20, "Empty")
|
||||
empty.query_context = raw_context
|
||||
mocks["get_charts_in_layout_order"].return_value = [good, empty]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {mocks["email"].ERROR_NO_QUERY_CONTEXT: ["20 - Empty"]}
|
||||
# The empty chart is skipped before any query runs; only the good one runs.
|
||||
mocks["ChartDataCommand"].return_value.run.assert_called_once()
|
||||
|
||||
|
||||
def test_empty_query_context_rebuilt_from_form_data_for_eligible_viz(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
# An eligible viz type (table) with no saved query context is rebuilt from
|
||||
# its form data and exported instead of being skipped.
|
||||
good = _chart(10, "Good")
|
||||
rebuilt = _chart(20, "Rebuilt", viz_type="table")
|
||||
rebuilt.query_context = None
|
||||
rebuilt.params = json.dumps({"groupby": ["country"], "metrics": ["count"]})
|
||||
rebuilt.datasource_id = 5
|
||||
rebuilt.datasource_type = "table"
|
||||
mocks["get_charts_in_layout_order"].return_value = [good, rebuilt]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {}
|
||||
# Both charts ran a query (the saved one and the rebuilt one).
|
||||
assert mocks["ChartDataCommand"].return_value.run.call_count == 2
|
||||
|
||||
|
||||
def test_empty_query_context_ineligible_viz_is_skipped(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
# A viz type outside the rebuild allowlist (mixed_timeseries — a multi-query
|
||||
# chart the generic rebuild can't reproduce) is skipped, not exported wrong.
|
||||
good = _chart(10, "Good")
|
||||
ineligible = _chart(20, "Ineligible", viz_type="mixed_timeseries")
|
||||
ineligible.query_context = None
|
||||
ineligible.params = json.dumps({"groupby": ["x"], "metrics": ["count"]})
|
||||
ineligible.datasource_id = 5
|
||||
mocks["get_charts_in_layout_order"].return_value = [good, ineligible]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {
|
||||
mocks["email"].ERROR_NO_QUERY_CONTEXT: ["20 - Ineligible"]
|
||||
}
|
||||
mocks["ChartDataCommand"].return_value.run.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("params", "datasource_id"),
|
||||
[
|
||||
("not valid json", 5), # params don't parse → cannot rebuild
|
||||
("null", 5), # params parse to a non-object → cannot rebuild
|
||||
('{"groupby": ["x"]}', None), # no datasource to point the query at
|
||||
],
|
||||
)
|
||||
def test_eligible_viz_skipped_when_form_data_unusable(
|
||||
mocks: dict[str, Any], params: str, datasource_id: int | None
|
||||
) -> None:
|
||||
# Even for an allowlisted viz type, a rebuild is only attempted when the form
|
||||
# data is a usable object and a datasource is known; otherwise the chart is
|
||||
# skipped rather than raising.
|
||||
good = _chart(10, "Good")
|
||||
bad = _chart(20, "Bad", viz_type="table")
|
||||
bad.query_context = None
|
||||
bad.params = params
|
||||
bad.datasource_id = datasource_id
|
||||
bad.datasource_type = "table"
|
||||
mocks["get_charts_in_layout_order"].return_value = [good, bad]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {mocks["email"].ERROR_NO_QUERY_CONTEXT: ["20 - Bad"]}
|
||||
# Only the good chart ran a query; the unusable one never reached execution.
|
||||
mocks["ChartDataCommand"].return_value.run.assert_called_once()
|
||||
|
||||
|
||||
def test_rebuilt_query_context_payload_carries_query_shape(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
# Assert the actual payload handed to ChartDataQueryContextSchema().load for a
|
||||
# rebuilt chart: columns, filters, ordering and granularity must survive so the
|
||||
# exported data matches the chart (not just that a query ran).
|
||||
chart = _chart(10, "Rebuilt", viz_type="table")
|
||||
chart.query_context = None
|
||||
chart.datasource_id = 5
|
||||
chart.datasource_type = "table"
|
||||
chart.params = json.dumps(
|
||||
{
|
||||
"groupby": ["country"],
|
||||
"metrics": ["count"],
|
||||
"granularity_sqla": "ds",
|
||||
"time_range": "Last quarter",
|
||||
"row_limit": 25,
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "year",
|
||||
"operator": ">",
|
||||
"comparator": 2000,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
mocks["get_charts_in_layout_order"].return_value = [chart]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
load_args = mocks["ChartDataQueryContextSchema"].return_value.load.call_args
|
||||
payload = load_args.args[0]
|
||||
query = payload["queries"][0]
|
||||
assert query["columns"] == ["country"]
|
||||
assert query["metrics"] == ["count"]
|
||||
assert query["filters"] == [{"col": "year", "op": ">", "val": 2000}]
|
||||
assert query["orderby"] == [["count", False]]
|
||||
assert query["granularity"] == "ds"
|
||||
assert query["time_range"] == "Last quarter"
|
||||
assert query["row_limit"] == 25
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
(None, {"table", "big_number_total", "big_number", "pie"}), # default
|
||||
(set(), set()), # explicit empty set disables rebuild entirely
|
||||
({"table"}, {"table"}), # explicit override
|
||||
],
|
||||
)
|
||||
def test_rebuild_viz_types_respects_explicit_empty_set(
|
||||
configured: set[str] | None, expected: set[str]
|
||||
) -> None:
|
||||
from superset.tasks import export_dashboard_excel as module
|
||||
|
||||
fake_app = mock.MagicMock()
|
||||
fake_app.config.get.return_value = configured
|
||||
with mock.patch.object(module, "current_app", fake_app):
|
||||
assert module._rebuild_viz_types() == expected
|
||||
|
||||
|
||||
def test_chart_query_error_grouped_as_general_export_continues(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user