mirror of
https://github.com/apache/superset.git
synced 2026-09-01 21:11:28 +00:00
5418 lines
221 KiB
Python
5418 lines
221 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
||
# or more contributor license agreements. See the NOTICE file
|
||
# distributed with this work for additional information
|
||
# regarding copyright ownership. The ASF licenses this file
|
||
# to you under the Apache License, Version 2.0 (the
|
||
# "License"); you may not use this file except in compliance
|
||
# with the License. You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing,
|
||
# software distributed under the License is distributed on an
|
||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
# KIND, either express or implied. See the License for the
|
||
# specific language governing permissions and limitations
|
||
# under the License.
|
||
# pylint: disable=too-many-lines
|
||
"""a collection of model-related helper classes and functions"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import ast
|
||
import builtins
|
||
import copy
|
||
import dataclasses
|
||
import logging
|
||
import re
|
||
import uuid
|
||
from collections.abc import Hashable, Iterator
|
||
from contextlib import contextmanager
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import (
|
||
Any,
|
||
Callable,
|
||
cast,
|
||
ClassVar,
|
||
Literal,
|
||
NamedTuple,
|
||
Optional,
|
||
TYPE_CHECKING,
|
||
TypedDict,
|
||
Union,
|
||
)
|
||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||
|
||
import dateutil.parser
|
||
import humanize
|
||
import numpy as np
|
||
import pandas as pd
|
||
import pytz
|
||
import sqlalchemy as sa
|
||
import yaml
|
||
from flask import current_app as app, g
|
||
from flask_appbuilder import Model
|
||
from flask_appbuilder.models.decorators import renders
|
||
from flask_appbuilder.models.mixins import AuditMixin
|
||
from flask_appbuilder.security.sqla.models import User
|
||
from flask_babel import get_locale, lazy_gettext as _
|
||
from jinja2.exceptions import TemplateError, UndefinedError
|
||
from markupsafe import escape, Markup
|
||
from pandas import DateOffset
|
||
from sqlalchemy import and_, Column, or_, UniqueConstraint
|
||
from sqlalchemy.exc import MultipleResultsFound
|
||
from sqlalchemy.ext.hybrid import hybrid_property
|
||
from sqlalchemy.orm import (
|
||
declared_attr,
|
||
Mapped,
|
||
Mapper,
|
||
Session,
|
||
validates,
|
||
with_loader_criteria,
|
||
)
|
||
from sqlalchemy.orm.session import ORMExecuteState
|
||
from sqlalchemy.sql.elements import ColumnElement, Grouping, literal_column, TextClause
|
||
from sqlalchemy.sql.expression import Label, Select, TextAsFrom
|
||
from sqlalchemy.sql.selectable import Alias, TableClause
|
||
from sqlalchemy_utils import UUIDType
|
||
|
||
from superset import db, is_feature_enabled
|
||
from superset.advanced_data_type.types import AdvancedDataTypeResponse
|
||
from superset.common.db_query_status import QueryStatus
|
||
from superset.common.grouping_sets import (
|
||
grouping_id_column,
|
||
grouping_marker_label,
|
||
grouping_sets_clause,
|
||
)
|
||
from superset.common.utils import dataframe_utils
|
||
from superset.common.utils.time_range_utils import (
|
||
get_since_until_from_query_object,
|
||
get_since_until_from_time_range,
|
||
)
|
||
from superset.constants import (
|
||
CacheRegion,
|
||
EMPTY_STRING,
|
||
NULL_STRING,
|
||
SKIP_VISIBILITY_FILTER_CLASSES,
|
||
TimeGrain,
|
||
)
|
||
from superset.db_engine_specs.base import TimestampExpression
|
||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||
from superset.exceptions import (
|
||
AdvancedDataTypeResponseError,
|
||
ColumnNotFoundException,
|
||
InvalidPostProcessingError,
|
||
QueryClauseValidationException,
|
||
QueryObjectValidationError,
|
||
SupersetDisallowedSQLFunctionException,
|
||
SupersetDisallowedSQLTableException,
|
||
SupersetErrorException,
|
||
SupersetErrorsException,
|
||
SupersetParseError,
|
||
SupersetSecurityException,
|
||
SupersetSyntaxErrorException,
|
||
)
|
||
from superset.extensions import feature_flag_manager
|
||
from superset.jinja_context import BaseTemplateProcessor
|
||
from superset.sql.parse import has_aggregate, sanitize_clause, SQLScript, SQLStatement
|
||
from superset.superset_typing import (
|
||
AdhocColumn,
|
||
AdhocMetric,
|
||
Column as ColumnTyping,
|
||
FilterValue,
|
||
FilterValues,
|
||
Metric,
|
||
OrderBy,
|
||
QueryObjectDict,
|
||
)
|
||
from superset.utils import core as utils, json
|
||
from superset.utils.core import (
|
||
DateColumn,
|
||
DTTM_ALIAS,
|
||
FilterOperator,
|
||
GenericDataType,
|
||
get_base_axis_columns,
|
||
get_base_axis_labels,
|
||
get_column_name,
|
||
get_column_names,
|
||
get_metric_names,
|
||
get_non_base_axis_columns,
|
||
get_user_id,
|
||
get_x_axis_label,
|
||
is_adhoc_column,
|
||
MediumText,
|
||
normalize_dttm_col,
|
||
QueryObjectFilterClause,
|
||
remove_duplicates,
|
||
SqlExpressionType,
|
||
TIME_COMPARISON,
|
||
)
|
||
from superset.utils.date_parser import (
|
||
get_past_or_future,
|
||
is_constant_human_timedelta,
|
||
is_parseable_human_timedelta,
|
||
normalize_time_delta,
|
||
TimeDeltaAmbiguousError,
|
||
)
|
||
from superset.utils.dates import datetime_to_epoch
|
||
from superset.utils.rls import apply_rls, get_predicates_for_table
|
||
|
||
|
||
class ValidationResultDict(TypedDict):
|
||
"""Type for validation result objects returned by validate_expression."""
|
||
|
||
valid: bool
|
||
errors: list[dict[str, Any]]
|
||
|
||
|
||
if TYPE_CHECKING:
|
||
from superset.common.query_object import QueryObject
|
||
from superset.connectors.sqla.models import SqlMetric, TableColumn
|
||
from superset.db_engine_specs import BaseEngineSpec
|
||
from superset.models.core import Database
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
VIRTUAL_TABLE_ALIAS = "virtual_table"
|
||
SERIES_LIMIT_SUBQ_ALIAS = "series_limit"
|
||
|
||
# Offset join column suffix used for joining offset results
|
||
OFFSET_JOIN_COLUMN_SUFFIX = "__offset_join_column_"
|
||
|
||
|
||
def get_effective_hours_offset(
|
||
db_engine_spec: type["BaseEngineSpec"],
|
||
column_type: str | None,
|
||
offset_hours: int,
|
||
db_extra: dict[str, Any] | None = None,
|
||
) -> int:
|
||
"""Return the dataset offset representable by a temporal column's type."""
|
||
sqla_type = db_engine_spec.get_sqla_column_type(column_type, db_extra=db_extra)
|
||
if isinstance(sqla_type, sa.Date):
|
||
# int() deliberately truncates toward zero; // would turn -1h into -24h.
|
||
return int(offset_hours / 24) * 24
|
||
return offset_hours
|
||
|
||
|
||
# Right suffix used for joining offset results
|
||
R_SUFFIX = "__right_suffix"
|
||
|
||
|
||
def _normalize_mssql_virtual_dataset_sql(
|
||
sql: str, parsed_script: SQLScript, engine: str
|
||
) -> str:
|
||
"""Remove SQL Server ordering that is invalid inside a derived table."""
|
||
if engine != "mssql" or not parsed_script.statements:
|
||
return sql
|
||
|
||
statement = parsed_script.statements[0]
|
||
if not isinstance(statement, SQLStatement):
|
||
return sql
|
||
|
||
return (
|
||
parsed_script.format()
|
||
if statement.remove_unbounded_top_level_order_by()
|
||
else sql
|
||
)
|
||
|
||
|
||
def _as_wall_clock(series: pd.Series) -> pd.Series:
|
||
"""
|
||
Return a datetime series as local wall-clock readings, dropping any
|
||
timezone. Series of other dtypes are returned unchanged.
|
||
"""
|
||
if isinstance(series.dtype, pd.DatetimeTZDtype):
|
||
return series.dt.tz_localize(None)
|
||
return series
|
||
|
||
|
||
class _TemporalColumnMetadata(NamedTuple):
|
||
"""Temporal metadata resolved from a physical dataset column."""
|
||
|
||
is_temporal: bool = False
|
||
python_date_format: str | None = None
|
||
|
||
|
||
def _get_temporal_physical_column_metadata(
|
||
datasource: Any, column_name: str | None
|
||
) -> _TemporalColumnMetadata:
|
||
"""Resolve temporal metadata using the physical column's precedence rules."""
|
||
if not column_name or not hasattr(datasource, "get_column"):
|
||
return _TemporalColumnMetadata()
|
||
column = datasource.get_column(column_name.strip())
|
||
if not column:
|
||
return _TemporalColumnMetadata()
|
||
if isinstance(column, dict):
|
||
declared_is_dttm = column.get("is_dttm")
|
||
is_temporal = (
|
||
bool(declared_is_dttm)
|
||
if declared_is_dttm is not None
|
||
else column.get("type_generic") == GenericDataType.TEMPORAL
|
||
)
|
||
python_date_format = column.get("python_date_format")
|
||
else:
|
||
is_temporal_property = getattr(column, "is_temporal", None)
|
||
if is_temporal_property is not None:
|
||
is_temporal = bool(is_temporal_property)
|
||
else:
|
||
declared_is_dttm = getattr(column, "is_dttm", None)
|
||
is_temporal = (
|
||
bool(declared_is_dttm)
|
||
if declared_is_dttm is not None
|
||
else getattr(column, "type_generic", None) == GenericDataType.TEMPORAL
|
||
)
|
||
python_date_format = getattr(column, "python_date_format", None)
|
||
return _TemporalColumnMetadata(
|
||
is_temporal=is_temporal,
|
||
python_date_format=(str(python_date_format) if python_date_format else None),
|
||
)
|
||
|
||
|
||
def _temporal_axis_parse_error(column_name: str) -> QueryObjectValidationError:
|
||
"""Build the user-facing error for an unparseable temporal join axis."""
|
||
return QueryObjectValidationError(
|
||
_(
|
||
"Unable to align time comparison because temporal axis "
|
||
"'%(column)s' contains values that cannot be parsed as datetimes. "
|
||
"Update the column's datetime format or choose a valid temporal "
|
||
"column.",
|
||
column=column_name,
|
||
)
|
||
)
|
||
|
||
|
||
def _apply_temporal_join_format(
|
||
series: pd.Series,
|
||
column_name: str,
|
||
datetime_format: str | None,
|
||
) -> pd.Series:
|
||
"""Apply a dataset column's declared datetime format to working values."""
|
||
if not datetime_format:
|
||
return series
|
||
working_df = pd.DataFrame({column_name: series.copy()})
|
||
normalize_dttm_col(
|
||
working_df,
|
||
(
|
||
DateColumn(
|
||
col_label=column_name,
|
||
timestamp_format=datetime_format,
|
||
),
|
||
),
|
||
)
|
||
return working_df[column_name]
|
||
|
||
|
||
def _parse_temporal_join_values(series: pd.Series, column_name: str) -> pd.Series:
|
||
"""Parse working temporal values and wrap pandas parser-policy errors."""
|
||
if pd.api.types.is_datetime64_any_dtype(series):
|
||
return series
|
||
try:
|
||
return pd.to_datetime(series, errors="coerce", format="mixed")
|
||
except (TypeError, ValueError) as ex:
|
||
raise _temporal_axis_parse_error(column_name) from ex
|
||
|
||
|
||
def _retry_temporal_join_values_at_wider_resolution(
|
||
series: pd.Series,
|
||
column_name: str,
|
||
datetime_format: str | None,
|
||
) -> pd.Series:
|
||
"""Retry valid values outside pandas' nanosecond datetime range."""
|
||
resolution = "ms" if datetime_format == "epoch_ms" else "s"
|
||
try:
|
||
if datetime_format and datetime_format not in {"epoch_s", "epoch_ms"}:
|
||
parsed_values = [
|
||
datetime.strptime(str(value), datetime_format)
|
||
if pd.notna(value)
|
||
else None
|
||
for value in series
|
||
]
|
||
else:
|
||
parsed_values = series
|
||
converted = pd.Series(
|
||
pd.array(parsed_values, dtype=f"datetime64[{resolution}]"),
|
||
index=series.index,
|
||
name=series.name,
|
||
)
|
||
except (OverflowError, TypeError, ValueError) as ex:
|
||
raise _temporal_axis_parse_error(column_name) from ex
|
||
if (series.notna() & converted.isna()).any():
|
||
raise _temporal_axis_parse_error(column_name)
|
||
return converted
|
||
|
||
|
||
def _coerce_temporal_join_series(
|
||
series: pd.Series,
|
||
column_name: str,
|
||
datetime_format: str | None = None,
|
||
) -> pd.Series:
|
||
"""Parse a temporal join series losslessly and normalize it to wall clocks."""
|
||
if series.isna().all():
|
||
return series
|
||
|
||
converted = _apply_temporal_join_format(series, column_name, datetime_format)
|
||
converted = _parse_temporal_join_values(converted, column_name)
|
||
if (series.notna() & converted.isna()).any():
|
||
converted = _retry_temporal_join_values_at_wider_resolution(
|
||
series, column_name, datetime_format
|
||
)
|
||
|
||
if isinstance(converted.dtype, pd.DatetimeTZDtype):
|
||
return converted.dt.tz_localize(None)
|
||
if pd.api.types.is_datetime64_any_dtype(converted):
|
||
return converted
|
||
|
||
def as_wall_clock(value: Any) -> pd.Timestamp:
|
||
if pd.isna(value):
|
||
return pd.NaT
|
||
timestamp = pd.Timestamp(value)
|
||
return timestamp.tz_localize(None) if timestamp.tzinfo else timestamp
|
||
|
||
return converted.map(as_wall_clock)
|
||
|
||
|
||
def _has_multiple_utc_offsets(series: pd.Series) -> bool:
|
||
"""Return whether every value is timezone-aware and offsets differ."""
|
||
utc_offsets: set[timedelta | None] = set()
|
||
for value in series.dropna():
|
||
try:
|
||
timestamp = pd.Timestamp(value)
|
||
except (TypeError, ValueError):
|
||
return False
|
||
if timestamp.tzinfo is None:
|
||
return False
|
||
utc_offsets.add(timestamp.utcoffset())
|
||
return len(utc_offsets) > 1
|
||
|
||
|
||
def _shift_grainless_temporal_source(
|
||
source: pd.Series,
|
||
offset: str,
|
||
delta: DateOffset | None,
|
||
) -> pd.Series:
|
||
"""Shift a temporal join source after validating free-form anchors."""
|
||
if delta is None and not is_constant_human_timedelta(offset):
|
||
raise QueryObjectValidationError(
|
||
_("Time Grain must be specified when using Time Comparison.")
|
||
)
|
||
if source.isna().all():
|
||
return source.map(lambda value: pd.NaT)
|
||
if delta is not None:
|
||
return source + delta
|
||
|
||
def shift(value: pd.Timestamp) -> pd.Timestamp:
|
||
if pd.isna(value):
|
||
return value
|
||
truncated = value.floor("s").to_pydatetime()
|
||
return value + (get_past_or_future(offset, truncated) - truncated)
|
||
|
||
return source.map(shift)
|
||
|
||
|
||
class CachedTimeOffset(TypedDict):
|
||
"""Result type for time offset processing"""
|
||
|
||
df: pd.DataFrame
|
||
queries: list[str]
|
||
cache_keys: list[str | None]
|
||
|
||
|
||
# Keys used to filter QueryObjectDict for get_sqla_query parameters
|
||
SQLA_QUERY_KEYS = {
|
||
"apply_fetch_values_predicate",
|
||
"columns",
|
||
"extras",
|
||
"filter",
|
||
"from_dttm",
|
||
"granularity",
|
||
"groupby",
|
||
"inner_from_dttm",
|
||
"inner_to_dttm",
|
||
"is_rowcount",
|
||
"is_timeseries",
|
||
"metrics",
|
||
"orderby",
|
||
"order_desc",
|
||
"to_dttm",
|
||
"series_columns",
|
||
"series_limit",
|
||
"series_limit_metric",
|
||
"group_others_when_limit_reached",
|
||
"grouping_sets",
|
||
"row_limit",
|
||
"row_offset",
|
||
"timeseries_limit",
|
||
"timeseries_limit_metric",
|
||
"time_shift",
|
||
}
|
||
|
||
|
||
def validate_adhoc_subquery(
|
||
sql: str,
|
||
database: Database,
|
||
catalog: str | None,
|
||
default_schema: str,
|
||
engine: str,
|
||
) -> str:
|
||
"""
|
||
Check if adhoc SQL contains sub-queries or nested sub-queries with table.
|
||
|
||
If sub-queries are allowed, the adhoc SQL is modified to insert any applicable RLS
|
||
predicates to it.
|
||
|
||
:param sql: adhoc sql expression
|
||
:raise SupersetSecurityException if sql contains sub-queries or
|
||
nested sub-queries with table
|
||
"""
|
||
parsed_statement = SQLStatement(sql, engine)
|
||
rls_applied = False
|
||
if parsed_statement.has_subquery():
|
||
if not is_feature_enabled("ALLOW_ADHOC_SUBQUERY"):
|
||
raise SupersetSecurityException(
|
||
SupersetError(
|
||
error_type=SupersetErrorType.ADHOC_SUBQUERY_NOT_ALLOWED_ERROR,
|
||
message=_("Custom SQL fields cannot contain sub-queries."),
|
||
level=ErrorLevel.ERROR,
|
||
)
|
||
)
|
||
|
||
# enforce RLS rules in any relevant tables
|
||
rls_applied = apply_rls(database, catalog, default_schema, parsed_statement)
|
||
|
||
# Only regenerate the SQL if RLS predicates were actually applied;
|
||
# unnecessary round-tripping through sqlglot can alter dialect-specific
|
||
# syntax.
|
||
return parsed_statement.format() if rls_applied else sql
|
||
|
||
|
||
def validate_stored_expression_at_query_time(
|
||
expression: str,
|
||
database: Database,
|
||
catalog: str | None,
|
||
schema: str | None,
|
||
engine: str,
|
||
) -> str:
|
||
"""
|
||
Validate a stored column/metric expression at the point of use, applying the
|
||
same sub-query policy and RLS injection as adhoc expressions. Only the
|
||
dataset update path validates on save; v1 import
|
||
(``superset/commands/dataset/importers/v1/``) and dataset duplication
|
||
(``superset/commands/dataset/duplicate.py``) persist expressions as-is, and
|
||
rows written before the save-time check was added were never validated at
|
||
all. Jinja templating also rewrites the expression after save. The query
|
||
sink is therefore the reliable place to enforce the policy.
|
||
|
||
A parse failure is ambiguous: the expression may use dialect-specific syntax
|
||
sqlglot cannot handle (e.g. ``DATE_ADD(ds, 1)`` on MySQL), it may be a
|
||
fragment that only parses inside a select list (``DISTINCT <expr>``), or it
|
||
may be unparseable precisely because something was appended to it. Before
|
||
falling back to the raw expression, the synthetic ``SELECT <expr>`` form and
|
||
the permissive dialect are used as detectors, so a sub-query or a second
|
||
statement sitting next to benign-but-unparseable syntax is still caught.
|
||
Only when no detector can parse the expression is validation skipped (and
|
||
logged), matching the pre-gate behaviour for such expressions.
|
||
|
||
A disallowed sub-query is surfaced as ``QueryObjectValidationError`` (a
|
||
chart-level 400) to match the adhoc sinks, rather than a raw 403.
|
||
"""
|
||
default_schema = schema or ""
|
||
try:
|
||
return validate_adhoc_subquery(
|
||
expression, database, catalog, default_schema, engine
|
||
)
|
||
except SupersetSecurityException as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
except SupersetParseError:
|
||
pass
|
||
|
||
# The detector passes below are detection only: the SQL they return is
|
||
# discarded rather than returned, since it is wrapped and may be in the wrong
|
||
# dialect. RLS injection is therefore not applied here, matching how such
|
||
# expressions reached the query before this check existed.
|
||
wrapped = f"SELECT {expression}"
|
||
for dialect in (engine, "base"):
|
||
try:
|
||
if len(SQLScript(wrapped, dialect).statements) > 1:
|
||
raise QueryObjectValidationError(
|
||
_("Custom SQL fields cannot be parsed as a single SQL statement.")
|
||
)
|
||
validate_adhoc_subquery(wrapped, database, catalog, default_schema, dialect)
|
||
except SupersetParseError:
|
||
continue
|
||
except SupersetSecurityException as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
return expression
|
||
|
||
logger.warning("Skipping query-time validation of unparseable stored expression")
|
||
return expression
|
||
|
||
|
||
def validate_rendered_expression(
|
||
expression: str,
|
||
database: Database,
|
||
catalog: str | None,
|
||
schema: str | None,
|
||
) -> str:
|
||
"""
|
||
Apply the stored-expression validation policy to a rendered expression.
|
||
|
||
Query-time counterpart to ``validate_stored_expression``: it runs on the
|
||
already-rendered expression that is embedded via ``literal_column`` and
|
||
applies the same policy, failing closed on unparseable results.
|
||
|
||
:param expression: the rendered expression
|
||
:returns: the expression to embed, possibly rewritten with RLS predicates
|
||
:raises QueryObjectValidationError: on multi-statement, set-operation,
|
||
disallowed sub-query, or sanitization failures -- matching the
|
||
``QueryObjectValidationError`` contract callers already expect from
|
||
``validate_stored_expression_at_query_time``, rather than letting a
|
||
raw ``SupersetSecurityException`` escape uncaught.
|
||
"""
|
||
engine = database.backend
|
||
wrapped = f"SELECT {expression}"
|
||
|
||
try:
|
||
parsed = SQLStatement(wrapped, engine)
|
||
except SupersetParseError as ex:
|
||
raise QueryObjectValidationError(
|
||
_("Custom SQL fields cannot be parsed as a single SQL statement.")
|
||
) from ex
|
||
|
||
if parsed.is_set_operation():
|
||
raise QueryObjectValidationError(
|
||
_("Custom SQL fields cannot contain set operations.")
|
||
)
|
||
|
||
try:
|
||
wrapped = validate_adhoc_subquery(
|
||
wrapped, database, catalog, schema or "", engine
|
||
)
|
||
except SupersetSecurityException as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
try:
|
||
wrapped = sanitize_clause(wrapped, engine)
|
||
except QueryClauseValidationException as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
|
||
prefix, expression = re.split(
|
||
r"SELECT\s+",
|
||
wrapped,
|
||
maxsplit=1,
|
||
flags=re.IGNORECASE,
|
||
)
|
||
return expression.strip()
|
||
|
||
|
||
def json_to_dict(json_str: str) -> dict[Any, Any]:
|
||
if json_str:
|
||
val = re.sub(",[ \t\r\n]+}", "}", json_str)
|
||
val = re.sub(",[ \t\r\n]+\\]", "]", val)
|
||
return json.loads(val)
|
||
|
||
return {}
|
||
|
||
|
||
UUID_NATIVE_TYPE_RE: re.Pattern[str] = re.compile(
|
||
r"\b(uuid|uniqueidentifier)\b", re.IGNORECASE
|
||
)
|
||
|
||
|
||
def parse_array_literal(value: Any) -> list[Any]:
|
||
"""
|
||
Parse a user-entered array literal (e.g. ``['a', 'b']`` or ``[1, 2]``) into a
|
||
list of elements, for the whole-array (column-level) array operators.
|
||
|
||
Accepts either an actual list/tuple, a bracketed literal string (parsed with
|
||
``ast.literal_eval``), or a plain scalar (wrapped into a single-element list).
|
||
Falls back to a single-element list when the string is not a valid literal.
|
||
"""
|
||
if isinstance(value, (list, tuple)):
|
||
return list(value)
|
||
if isinstance(value, str):
|
||
stripped = value.strip()
|
||
if stripped.startswith("[") and stripped.endswith("]"):
|
||
try:
|
||
parsed = ast.literal_eval(stripped)
|
||
except (ValueError, SyntaxError):
|
||
parsed = None
|
||
if isinstance(parsed, (list, tuple)):
|
||
return list(parsed)
|
||
return [value]
|
||
|
||
|
||
def coerce_array_values(
|
||
values: list[Any], element_type: Optional[utils.GenericDataType]
|
||
) -> list[Any]:
|
||
"""
|
||
Coerce array-element ``values`` to the array column's element type so the
|
||
emitted literal matches the column. Array columns map to a SQLAlchemy
|
||
``String`` type, so values arrive as strings and would otherwise build
|
||
string literals (e.g. ``array('5')``) that fail against a numeric array on
|
||
the server. Numeric elements are cast to numbers and boolean elements to
|
||
booleans; every other element type (string, temporal, enum, unknown) is left
|
||
untouched.
|
||
|
||
:param values: element values entered for an array filter
|
||
:param element_type: the array's element :class:`GenericDataType`, or None
|
||
:return: the coerced values
|
||
"""
|
||
if element_type == utils.GenericDataType.NUMERIC:
|
||
return [utils.cast_to_num(v) if isinstance(v, str) else v for v in values]
|
||
if element_type == utils.GenericDataType.BOOLEAN:
|
||
return [utils.cast_to_boolean(v) if isinstance(v, str) else v for v in values]
|
||
return values
|
||
|
||
|
||
def is_uuid_native_type(native_type: Optional[str]) -> bool:
|
||
"""
|
||
Return True if a native column type represents a UUID.
|
||
|
||
Engines such as PostgreSQL and ClickHouse expose native UUID column types
|
||
(e.g. ``UUID``, ``Nullable(UUID)``) that map to ``GenericDataType.STRING``
|
||
yet reject LIKE/ILIKE against the raw column, so these columns need an
|
||
explicit cast to string before pattern matching. SQL Server's equivalent
|
||
native type is ``uniqueidentifier``, which is matched too. The match is
|
||
on whole words so unrelated types that merely contain one of these as a
|
||
substring (e.g. a hypothetical ``uuidish`` type) aren't misclassified.
|
||
"""
|
||
return native_type is not None and bool(
|
||
UUID_NATIVE_TYPE_RE.search(native_type.strip())
|
||
)
|
||
|
||
|
||
def convert_uuids(obj: Any) -> Any:
|
||
"""
|
||
Convert UUID objects to str so we can use yaml.safe_dump
|
||
"""
|
||
if isinstance(obj, uuid.UUID):
|
||
return str(obj)
|
||
|
||
if isinstance(obj, list):
|
||
return [convert_uuids(el) for el in obj]
|
||
|
||
if isinstance(obj, dict):
|
||
return {k: convert_uuids(v) for k, v in obj.items()}
|
||
|
||
return obj
|
||
|
||
|
||
class UUIDMixin: # pylint: disable=too-few-public-methods
|
||
uuid = sa.Column(
|
||
UUIDType(binary=True), primary_key=False, unique=True, default=uuid.uuid4
|
||
)
|
||
|
||
@validates("uuid")
|
||
def _coerce_uuid(self, key: str, value: Any) -> Any: # noqa: ARG002
|
||
"""Coerce well-formed UUID strings to ``uuid.UUID`` on assignment;
|
||
pass everything else through untouched.
|
||
|
||
**Why coerce.** ``UUIDType`` only converts at SQL bind / SQL
|
||
result time. Importers and ad-hoc construction
|
||
(``SqlMetric(uuid="…string…")``) leave the in-memory attribute
|
||
as a ``str`` until the next DB round-trip refreshes it. With
|
||
SQLAlchemy-Continuum versioning attached to a child mapper
|
||
(``TableColumn`` / ``SqlMetric``), the post-INSERT attribute-
|
||
expire behaviour changes enough that the refresh doesn't happen
|
||
before the caller reads the attribute — breaking equality
|
||
assertions like ``test_import_dataset``'s
|
||
``metric.uuid == uuid.UUID(...)`` because str ≠ UUID. Coercing
|
||
defensively here makes the in-memory attribute always a UUID
|
||
regardless of provenance.
|
||
|
||
**Why the non-UUID-string escape hatch.** Tightening this
|
||
validator to raise on non-UUID strings would break a small set
|
||
of existing unit tests that use human-readable placeholder
|
||
strings as fixture uuids (e.g.
|
||
``test_dashboard_schemas.py``'s ``"dashboard-uuid-7"`` and
|
||
analogous placeholders in importer tests). The fixtures use
|
||
these placeholders for legibility — they're only ever compared
|
||
by string equality, never written to a real database. Letting
|
||
them through unchanged keeps the fixtures working at the cost
|
||
of deferring "real" UUID malformation to the SQL bind layer,
|
||
which raises a clearer "invalid input syntax for type uuid"
|
||
error keyed to the actual column.
|
||
|
||
**Tightening path** (if amin M1 is ever revisited): replace
|
||
the ``return value`` in the ``except`` branch with
|
||
``raise ValueError(f"Invalid UUID: {value!r}")``, then run the
|
||
unit test suite and migrate any remaining placeholder fixtures
|
||
to ``uuid.uuid4()`` (use
|
||
``rg '''SqlMetric\\(uuid="[^"]*"|"dashboard-uuid|"slice-uuid'''``
|
||
to find them). The full migration touches ~5–10 fixture files
|
||
and is non-breaking outside tests.
|
||
"""
|
||
if isinstance(value, str):
|
||
try:
|
||
return uuid.UUID(value)
|
||
except ValueError:
|
||
return value
|
||
return value
|
||
|
||
@property
|
||
def short_uuid(self) -> str:
|
||
return str(self.uuid)[:8]
|
||
|
||
|
||
class ChildMultipleResultsFound(MultipleResultsFound):
|
||
"""
|
||
An ambiguous lookup raised while importing a *child* object.
|
||
|
||
``ImportExportMixin.import_from_dict`` matches an incoming payload against
|
||
existing rows with a disjunction (unique-constraint match OR uuid match), so
|
||
a single payload entry can match two different DB rows — one by name and one
|
||
by uuid. When that happens for a top-level object it is raised before any
|
||
mutation, and callers may legitimately recover by skipping or returning the
|
||
existing row (see ``superset/commands/importers/v1/examples.py``).
|
||
|
||
When it happens for a *child* (``parent is not None``) the parent's fields
|
||
have already been updated and earlier siblings have already been imported,
|
||
so recovering by "returning the existing row" would report success over a
|
||
half-applied import. This subclass lets callers tell the two cases apart and
|
||
abort the transaction for the child case. It intentionally subclasses
|
||
``MultipleResultsFound`` so existing ``except MultipleResultsFound`` handlers
|
||
keep working.
|
||
"""
|
||
|
||
|
||
class ImportExportMixin(UUIDMixin):
|
||
export_parent: Optional[str] = None
|
||
# The name of the attribute
|
||
# with the SQL Alchemy back reference
|
||
|
||
export_children: list[str] = []
|
||
# List of (str) names of attributes
|
||
# with the SQL Alchemy forward references
|
||
|
||
export_fields: list[str] = []
|
||
# The names of the attributes
|
||
# that are available for import and export
|
||
|
||
extra_import_fields: list[str] = []
|
||
# Additional fields that should be imported,
|
||
# even though they were not exported
|
||
|
||
__mapper__: Mapper
|
||
|
||
@classmethod
|
||
def _unique_constraints(cls) -> list[set[str]]:
|
||
"""Get all (single column and multi column) unique constraints"""
|
||
unique = [
|
||
{c.name for c in u.columns}
|
||
for u in cls.__table_args__ # type: ignore
|
||
if isinstance(u, UniqueConstraint)
|
||
]
|
||
unique.extend({c.name} for c in cls.__table__.columns if c.unique) # type: ignore
|
||
return unique
|
||
|
||
@classmethod
|
||
def parent_foreign_key_mappings(cls) -> dict[str, str]:
|
||
"""Get a mapping of foreign name to the local name of foreign keys"""
|
||
parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
|
||
if parent_rel:
|
||
return {
|
||
local.name: remote.name
|
||
for (local, remote) in parent_rel.local_remote_pairs
|
||
}
|
||
return {}
|
||
|
||
@classmethod
|
||
def export_schema(
|
||
cls, recursive: bool = True, include_parent_ref: bool = False
|
||
) -> dict[str, Any]:
|
||
"""Export schema as a dictionary"""
|
||
parent_excludes = set()
|
||
if not include_parent_ref:
|
||
parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
|
||
if parent_ref:
|
||
parent_excludes = {column.name for column in parent_ref.local_columns}
|
||
|
||
def formatter(column: sa.Column) -> str:
|
||
return (
|
||
f"{str(column.type)} Default ({column.default.arg})"
|
||
if column.default
|
||
else str(column.type)
|
||
)
|
||
|
||
schema: dict[str, Any] = {
|
||
column.name: formatter(column)
|
||
for column in cls.__table__.columns # type: ignore
|
||
if (column.name in cls.export_fields and column.name not in parent_excludes)
|
||
}
|
||
if recursive:
|
||
for column in cls.export_children:
|
||
child_class = cls.__mapper__.relationships[column].argument.class_
|
||
schema[column] = [
|
||
child_class.export_schema(
|
||
recursive=recursive, include_parent_ref=include_parent_ref
|
||
)
|
||
]
|
||
return schema
|
||
|
||
@classmethod
|
||
def import_from_dict( # noqa: C901
|
||
# pylint: disable=too-many-arguments,too-many-branches,too-many-locals
|
||
cls,
|
||
dict_rep: dict[Any, Any],
|
||
parent: Optional[Any] = None,
|
||
recursive: bool = True,
|
||
sync: Optional[list[str]] = None,
|
||
allow_reparenting: bool = False,
|
||
) -> Any:
|
||
"""Import obj from a dictionary"""
|
||
if sync is None:
|
||
sync = []
|
||
parent_refs = cls.parent_foreign_key_mappings()
|
||
export_fields = (
|
||
set(cls.export_fields)
|
||
| set(cls.extra_import_fields)
|
||
| set(parent_refs.keys())
|
||
| {"uuid"}
|
||
)
|
||
new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
|
||
unique_constraints = cls._unique_constraints()
|
||
|
||
filters = [] # Using these filters to check if obj already exists
|
||
|
||
# Remove fields that should not get imported
|
||
for k in list(dict_rep):
|
||
if k not in export_fields and k not in parent_refs:
|
||
del dict_rep[k]
|
||
|
||
if not parent:
|
||
if cls.export_parent:
|
||
for prnt in parent_refs.keys():
|
||
if prnt not in dict_rep:
|
||
raise RuntimeError(f"{cls.__name__}: Missing field {prnt}")
|
||
else:
|
||
# Set foreign keys to parent obj
|
||
for k, v in parent_refs.items():
|
||
dict_rep[k] = getattr(parent, v)
|
||
|
||
if not allow_reparenting:
|
||
# Add filter for parent obj
|
||
filters.extend(
|
||
[getattr(cls, k) == dict_rep.get(k) for k in parent_refs.keys()]
|
||
)
|
||
|
||
# Add filter for unique constraints
|
||
ucs = [
|
||
and_(
|
||
*[
|
||
getattr(cls, k) == dict_rep.get(k)
|
||
for k in cs
|
||
if dict_rep.get(k) is not None
|
||
]
|
||
)
|
||
for cs in unique_constraints
|
||
]
|
||
filters.append(or_(*ucs))
|
||
|
||
# Check if object already exists in DB, break if more than one is found
|
||
try:
|
||
obj_query = db.session.query(cls).filter(and_(*filters))
|
||
obj = obj_query.one_or_none()
|
||
except MultipleResultsFound as ex:
|
||
# Now that children carry a ``uuid``, this match is a disjunction
|
||
# (name-match OR uuid-match), so a payload child can match one DB row
|
||
# by name and a different one by uuid — e.g. metrics renamed/swapped
|
||
# in the UI, then an older export re-imported with overwrite.
|
||
#
|
||
# That stays a hard error on purpose. Resolving it by preferring the
|
||
# uuid match would rename the uuid-matched row while the name-matched
|
||
# row still holds that name, trading this clear failure for an opaque
|
||
# ``UNIQUE constraint failed: (table_id, <name>)`` at the next flush.
|
||
#
|
||
# The two cases are not equally recoverable, so they are raised as
|
||
# distinct types:
|
||
#
|
||
# * ``parent is None`` (top-level object) — nothing has been mutated
|
||
# yet, so a caller may legitimately skip or fall back to the
|
||
# existing row. See the legacy NULL-schema handling in
|
||
# ``superset/commands/dataset/importers/v1/utils.py`` (including
|
||
# its ``deleted_at`` rollback) and the ``continue`` in
|
||
# ``superset/commands/importers/v1/examples.py`` (issue #16051).
|
||
# * ``parent is not None`` (child object) — the parent's scalar
|
||
# fields were already updated and earlier siblings were already
|
||
# imported, so there is no "unmodified row" to fall back to.
|
||
# Raising ``ChildMultipleResultsFound`` lets the dataset importer
|
||
# fail the import atomically (the command's transaction rolls
|
||
# back) instead of reporting success over a half-applied state.
|
||
logger.error(
|
||
"Error importing %s \n %s \n %s",
|
||
cls.__name__,
|
||
str(obj_query),
|
||
yaml.safe_dump(dict_rep),
|
||
exc_info=True,
|
||
)
|
||
if parent is not None:
|
||
raise ChildMultipleResultsFound(
|
||
f"{cls.__name__} matches more than one existing row under "
|
||
f"its parent {type(parent).__name__}"
|
||
) from ex
|
||
raise
|
||
|
||
# A child ``uuid`` is globally unique, but the match above is scoped to
|
||
# this ``parent`` (and also matches on name). Importing a config as a
|
||
# clone — e.g. a dataset re-imported under an edited uuid while the
|
||
# original still exists — can leave the incoming child uuid owned by a
|
||
# different row. Writing it, whether on INSERT (new obj) or on an
|
||
# overwrite UPDATE (obj matched by name), would violate the ``uuid``
|
||
# unique constraint at flush. Drop the incoming uuid whenever it belongs
|
||
# to some other row so ``UUIDMixin`` keeps/assigns a distinct one,
|
||
# reverting to the pre-uuid-export behavior for that clone.
|
||
if parent is not None and "uuid" in dict_rep:
|
||
if dict_rep["uuid"] is None:
|
||
# The child import schemas accept ``uuid: null``. On the
|
||
# overwrite UPDATE that would write a literal NULL over an
|
||
# existing child's uuid — silently, since the column is nullable
|
||
# and ``unique`` permits repeated NULLs — leaving a child no
|
||
# folder can reference and no export can round-trip.
|
||
del dict_rep["uuid"]
|
||
else:
|
||
# Known limitation: this lookup goes through the normal session
|
||
# visibility filter, so for child classes that are soft-deletable
|
||
# it cannot see a soft-deleted owner of the incoming uuid. In
|
||
# practice the children that matter here (``TableColumn`` /
|
||
# ``SqlMetric``) are not soft-deletable; the only soft-deletable
|
||
# child registered anywhere is ``SqlaTable`` via ``Database``'s
|
||
# ``export_children = ["tables"]`` (the v0 legacy path), which is
|
||
# reachable only from a hand-crafted v0 file. There a colliding
|
||
# uuid owned by a soft-deleted dataset would be kept and fail on
|
||
# the unique constraint at flush rather than being dropped here.
|
||
#
|
||
# Skip the lookup on the common idempotent re-import, where the
|
||
# name-matched ``obj`` already owns the incoming uuid: the guard
|
||
# would keep it anyway, so the query is pure waste on that path.
|
||
if obj is None or str(obj.uuid) != str(dict_rep["uuid"]):
|
||
uuid_owner = (
|
||
db.session.query(cls)
|
||
.filter(cls.uuid == dict_rep["uuid"])
|
||
.first()
|
||
)
|
||
if uuid_owner is not None and uuid_owner is not obj:
|
||
del dict_rep["uuid"]
|
||
|
||
if not obj:
|
||
is_new_obj = True
|
||
# Create new DB object
|
||
obj = cls(**dict_rep)
|
||
logger.debug("Importing new %s %s", obj.__tablename__, str(obj))
|
||
if cls.export_parent and parent:
|
||
setattr(obj, cls.export_parent, parent)
|
||
db.session.add(obj)
|
||
else:
|
||
is_new_obj = False
|
||
logger.debug("Updating %s %s", obj.__tablename__, str(obj))
|
||
# Update columns
|
||
for k, v in dict_rep.items():
|
||
setattr(obj, k, v)
|
||
|
||
# Recursively create children
|
||
if recursive:
|
||
for child in cls.export_children:
|
||
argument = cls.__mapper__.relationships[child].argument
|
||
child_class = (
|
||
argument.class_ if hasattr(argument, "class_") else argument
|
||
)
|
||
added = []
|
||
for c_obj in new_children.get(child, []):
|
||
added.append(
|
||
child_class.import_from_dict(
|
||
dict_rep=c_obj, parent=obj, sync=sync
|
||
)
|
||
)
|
||
# If children should get synced, delete the ones that did not
|
||
# get updated.
|
||
if child in sync and not is_new_obj:
|
||
back_refs = child_class.parent_foreign_key_mappings()
|
||
delete_filters = [
|
||
getattr(child_class, k) == getattr(obj, back_refs.get(k))
|
||
for k in back_refs.keys()
|
||
]
|
||
to_delete = set(
|
||
db.session.query(child_class).filter(and_(*delete_filters))
|
||
).difference(set(added))
|
||
for o in to_delete:
|
||
logger.debug("Deleting %s %s", child, str(obj))
|
||
db.session.delete(o)
|
||
|
||
return obj
|
||
|
||
def export_to_dict(
|
||
self,
|
||
recursive: bool = True,
|
||
include_parent_ref: bool = False,
|
||
include_defaults: bool = False,
|
||
export_uuids: bool = False,
|
||
) -> dict[Any, Any]:
|
||
"""Export obj to dictionary"""
|
||
export_fields = set(self.export_fields)
|
||
if export_uuids:
|
||
export_fields.add("uuid")
|
||
if "id" in export_fields:
|
||
export_fields.remove("id")
|
||
|
||
cls = self.__class__
|
||
parent_excludes = set()
|
||
if recursive and not include_parent_ref:
|
||
parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
|
||
if parent_ref:
|
||
parent_excludes = {c.name for c in parent_ref.local_columns}
|
||
dict_rep = {
|
||
# Convert c.name to str to handle SQLAlchemy's quoted_name type
|
||
# which is not YAML-serializable
|
||
str(c.name): getattr(self, c.name)
|
||
for c in cls.__table__.columns # type: ignore
|
||
if (
|
||
c.name in export_fields
|
||
and c.name not in parent_excludes
|
||
and (
|
||
include_defaults
|
||
or (
|
||
getattr(self, c.name) is not None
|
||
and (not c.default or getattr(self, c.name) != c.default.arg)
|
||
)
|
||
)
|
||
)
|
||
}
|
||
|
||
# sort according to export_fields using DSU (decorate, sort, undecorate)
|
||
order = {field: i for i, field in enumerate(self.export_fields)}
|
||
decorated_keys = [(order.get(k, len(order)), k) for k in dict_rep]
|
||
decorated_keys.sort()
|
||
dict_rep = {k: dict_rep[k] for _, k in decorated_keys}
|
||
|
||
if recursive:
|
||
for cld in self.export_children:
|
||
# sorting to make lists of children stable
|
||
dict_rep[cld] = sorted(
|
||
[
|
||
child.export_to_dict(
|
||
recursive=recursive,
|
||
include_parent_ref=include_parent_ref,
|
||
include_defaults=include_defaults,
|
||
export_uuids=export_uuids,
|
||
)
|
||
for child in getattr(self, cld)
|
||
],
|
||
key=lambda k: sorted(str(k.items())),
|
||
)
|
||
|
||
return convert_uuids(dict_rep)
|
||
|
||
def override(self, obj: Any) -> None:
|
||
"""Overrides the plain fields of the dashboard."""
|
||
for field in obj.__class__.export_fields:
|
||
setattr(self, field, getattr(obj, field))
|
||
|
||
def copy(self) -> Any:
|
||
"""Creates a copy of the dashboard without relationships."""
|
||
new_obj = self.__class__()
|
||
new_obj.override(self)
|
||
return new_obj
|
||
|
||
def alter_params(self, **kwargs: Any) -> None:
|
||
params = self.params_dict
|
||
params.update(kwargs)
|
||
self.params = json.dumps(params)
|
||
|
||
def remove_params(self, param_to_remove: str) -> None:
|
||
params = self.params_dict
|
||
params.pop(param_to_remove, None)
|
||
self.params = json.dumps(params)
|
||
|
||
def reset_ownership(self) -> None:
|
||
"""object will belong to the current user"""
|
||
from superset.subjects.utils import (
|
||
get_default_viewers_for_new_asset,
|
||
get_user_subject,
|
||
)
|
||
|
||
# Reset the audit pointers. When a Flask request context is
|
||
# available we explicitly stamp the current user, otherwise we
|
||
# leave the attributes unset so Flask-AppBuilder's column
|
||
# defaults fill them in on save. An explicit assignment is
|
||
# required because once the ``created_by`` / ``changed_by``
|
||
# relationships are configured (which happens eagerly on models
|
||
# registered with SQLAlchemy-Continuum), setting them to
|
||
# ``None`` propagates to the FK column and suppresses the
|
||
# ``default=`` callable.
|
||
self.editors = []
|
||
if g and hasattr(g, "user") and g.user:
|
||
self.created_by = g.user
|
||
self.changed_by = g.user
|
||
user_subject = get_user_subject(g.user.id)
|
||
if user_subject:
|
||
self.editors = [user_subject]
|
||
# Only dashboards and charts have ``viewers``; this mixin also
|
||
# serves datasets, which have editors only. Defaults never replace
|
||
# viewers already present on the instance, matching the create
|
||
# commands' "explicit viewers win" rule.
|
||
if hasattr(self, "viewers") and not self.viewers: # type: ignore[has-type]
|
||
self.viewers = get_default_viewers_for_new_asset(g.user.id)
|
||
else:
|
||
self.created_by = None
|
||
self.changed_by = None
|
||
|
||
@property
|
||
def params_dict(self) -> dict[Any, Any]:
|
||
return json_to_dict(self.params)
|
||
|
||
@property
|
||
def template_params_dict(self) -> dict[Any, Any]:
|
||
return json_to_dict(self.template_params) # type: ignore
|
||
|
||
|
||
def _user(user: User) -> str:
|
||
if not user:
|
||
return ""
|
||
return escape(user)
|
||
|
||
|
||
def format_time_humanized(timestamp: datetime) -> str:
|
||
"""Humanize *timestamp* against the server's naive-local clock.
|
||
|
||
Module-level rather than a mixin method so values projected outside an
|
||
entity (the archive list reads ``deleted_at`` in a bare column query) can
|
||
be humanized identically. The subtraction uses ``datetime.now()`` because
|
||
the audit columns and ``deleted_at`` are stamped with it; humanizing here,
|
||
on the clock that did the stamping, is what spares every client from
|
||
guessing the server's timezone.
|
||
"""
|
||
locale = str(get_locale())
|
||
time_diff = datetime.now() - timestamp
|
||
# Skip activation for 'en' locale as it's humanize's default locale
|
||
if locale == "en":
|
||
return humanize.naturaltime(time_diff)
|
||
try:
|
||
humanize.i18n.activate(locale)
|
||
try:
|
||
return humanize.naturaltime(time_diff)
|
||
finally:
|
||
# humanize's activation is thread-local, so there is no
|
||
# cross-request contamination between workers -- but without the
|
||
# finally, a naturaltime failure would leave the locale active
|
||
# for whatever request this thread serves next.
|
||
humanize.i18n.deactivate()
|
||
except Exception as e:
|
||
logger.warning("Locale '%s' is not supported in humanize: %s", locale, e)
|
||
return humanize.naturaltime(time_diff)
|
||
|
||
|
||
class AuditMixinNullable(AuditMixin):
|
||
"""Altering the AuditMixin to use nullable fields
|
||
|
||
Allows creating objects programmatically outside of CRUD
|
||
"""
|
||
|
||
created_on = sa.Column(sa.DateTime, default=datetime.now, nullable=True)
|
||
changed_on = sa.Column(
|
||
sa.DateTime, default=datetime.now, onupdate=datetime.now, nullable=True
|
||
)
|
||
|
||
@declared_attr
|
||
def created_by_fk(self) -> Mapped[Optional[int]]: # pylint: disable=arguments-renamed
|
||
return sa.Column(
|
||
sa.Integer,
|
||
sa.ForeignKey("ab_user.id"),
|
||
default=get_user_id,
|
||
nullable=True,
|
||
)
|
||
|
||
@declared_attr
|
||
def changed_by_fk(self) -> Mapped[Optional[int]]: # pylint: disable=arguments-renamed
|
||
return sa.Column(
|
||
sa.Integer,
|
||
sa.ForeignKey("ab_user.id"),
|
||
default=get_user_id,
|
||
onupdate=get_user_id,
|
||
nullable=True,
|
||
)
|
||
|
||
@property
|
||
def created_by_name(self) -> str:
|
||
if self.created_by:
|
||
return escape(f"{self.created_by}")
|
||
return ""
|
||
|
||
@property
|
||
def changed_by_name(self) -> str:
|
||
if self.changed_by:
|
||
return escape(f"{self.changed_by}")
|
||
return ""
|
||
|
||
@renders("created_by")
|
||
def creator(self) -> Union[Markup, str]:
|
||
return _user(self.created_by)
|
||
|
||
@property
|
||
def changed_by_(self) -> Union[Markup, str]:
|
||
return _user(self.changed_by)
|
||
|
||
@renders("changed_on")
|
||
def changed_on_(self) -> Markup:
|
||
return Markup(f'<span class="no-wrap">{self.changed_on}</span>') # noqa: S704
|
||
|
||
@renders("changed_on")
|
||
def changed_on_delta_humanized(self) -> str:
|
||
return self.changed_on_humanized
|
||
|
||
@renders("changed_on")
|
||
def changed_on_dttm(self) -> float:
|
||
return datetime_to_epoch(self.changed_on)
|
||
|
||
@renders("created_on")
|
||
def created_on_delta_humanized(self) -> str:
|
||
return self.created_on_humanized
|
||
|
||
@renders("changed_on")
|
||
def changed_on_utc(self) -> str:
|
||
# Convert naive datetime to UTC
|
||
return self.changed_on.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
|
||
|
||
def _format_time_humanized(self, timestamp: datetime) -> str:
|
||
return format_time_humanized(timestamp)
|
||
|
||
@property
|
||
def changed_on_humanized(self) -> str:
|
||
return self._format_time_humanized(self.changed_on)
|
||
|
||
@property
|
||
def created_on_humanized(self) -> str:
|
||
return self._format_time_humanized(self.created_on)
|
||
|
||
@renders("changed_on")
|
||
def modified(self) -> Markup:
|
||
return Markup(f'<span class="no-wrap">{self.changed_on_humanized}</span>') # noqa: S704
|
||
|
||
|
||
# Shared sentinel for "no bypass requested" — returned by
|
||
# ``_collect_bypass_classes`` on the common path so every primary SELECT
|
||
# does not allocate a fresh empty set. ``frozenset`` so accidental
|
||
# mutation by a caller is a TypeError, not silent corruption.
|
||
_NO_BYPASS: frozenset[type] = frozenset()
|
||
|
||
|
||
class SoftDeleteMixin:
|
||
"""Mixin that adds soft-delete support to a SQLAlchemy model.
|
||
|
||
Adds a nullable ``deleted_at`` column. When set, the row is treated as
|
||
deleted and excluded from standard ORM queries via a global
|
||
``do_orm_execute`` listener registered at app init.
|
||
|
||
Delete commands route through ``BaseDAO.delete()``, which detects
|
||
the mixin and calls ``soft_delete()`` to mark the row as deleted
|
||
(without removing it). ``BaseDAO.hard_delete()`` is the permanent
|
||
hard-deletion path; it bypasses the mixin and calls
|
||
``session.delete()`` directly.
|
||
|
||
The listener can be bypassed per-entity, either for one statement (via
|
||
``execution_options``) or for a session-scoped block (via
|
||
``session.info`` / the ``skip_visibility_filter`` context manager).
|
||
See ``_add_soft_delete_filter`` for the precise semantics.
|
||
|
||
Subclass registry: every concrete subclass registers itself in
|
||
``_registered_subclasses`` via ``__init_subclass__``. The listener
|
||
iterates this cached list rather than walking ``__subclasses__()``
|
||
on every primary SELECT — important because the listener fires on
|
||
every ORM query in the app, and the walk grows with each adopted
|
||
entity.
|
||
"""
|
||
|
||
_registered_subclasses: ClassVar[list[type[SoftDeleteMixin]]] = []
|
||
|
||
def __init_subclass__(cls, **kwargs: Any) -> None:
|
||
super().__init_subclass__(**kwargs)
|
||
# Cache the subclass once, sorted by qualname so the listener's
|
||
# ``with_loader_criteria`` options attach in a deterministic
|
||
# order across processes (stable compiled-statement cache key).
|
||
if cls in SoftDeleteMixin._registered_subclasses:
|
||
return
|
||
SoftDeleteMixin._registered_subclasses.append(cls)
|
||
SoftDeleteMixin._registered_subclasses.sort(key=lambda c: c.__qualname__)
|
||
|
||
deleted_at = sa.Column(sa.DateTime, nullable=True, index=True)
|
||
|
||
@hybrid_property
|
||
def is_deleted(self) -> bool:
|
||
return self.deleted_at is not None
|
||
|
||
@is_deleted.expression # type: ignore
|
||
def is_deleted(cls) -> ColumnElement: # noqa: N805
|
||
return cls.deleted_at.is_not(None)
|
||
|
||
@classmethod
|
||
def where_not_deleted(cls) -> ColumnElement:
|
||
"""Return a SQL WHERE clause that excludes soft-deleted rows.
|
||
|
||
Returns a ``ColumnElement`` (a SQL expression), not a Python bool —
|
||
use this in a query's ``.filter(...)`` call. The name reflects the
|
||
intent ("WHERE NOT deleted") rather than reading like a predicate.
|
||
"""
|
||
return cls.deleted_at.is_(None)
|
||
|
||
def soft_delete(self) -> None:
|
||
"""Mark this object as soft-deleted."""
|
||
# Naive datetime, mirroring AuditMixinNullable.changed_on. PR #33693
|
||
# reverted a UTC migration on the audit columns; if/when those move
|
||
# to UTC-aware, this assignment should follow.
|
||
self.deleted_at = datetime.now()
|
||
|
||
def restore(self) -> None:
|
||
"""Clear the soft-delete marker, making this object active again."""
|
||
self.deleted_at = None
|
||
|
||
|
||
def _collect_bypass_classes(execute_state: ORMExecuteState) -> frozenset[type]:
|
||
"""Union of bypass class sets from per-query and per-session sources.
|
||
|
||
Per-query: ``execution_options[SKIP_VISIBILITY_FILTER_CLASSES]`` — set
|
||
by ``BaseDAO.find_by_id(skip_visibility_filter=True)``,
|
||
``find_existing_for_import``, ``raise_for_editorship``, etc., for
|
||
narrow one-statement bypass.
|
||
|
||
Per-session: ``session.info[SKIP_VISIBILITY_FILTER_CLASSES]`` — set by
|
||
``BaseDeletedStateFilter.apply`` and the ``skip_visibility_filter``
|
||
context manager, for request-scoped bypass across multiple queries
|
||
issued on the same session (e.g., FAB list endpoints that build a
|
||
count query plus an inner+outer pair, where per-query options are
|
||
stripped between count and outer fetch).
|
||
|
||
Returns the shared empty ``_NO_BYPASS`` sentinel when neither source
|
||
has a bypass set — the common case for every ORM SELECT in the app —
|
||
so the listener's hot path does not allocate per-call empty sets.
|
||
"""
|
||
per_query = execute_state.execution_options.get(SKIP_VISIBILITY_FILTER_CLASSES)
|
||
per_session = execute_state.session.info.get(SKIP_VISIBILITY_FILTER_CLASSES)
|
||
if not per_query and not per_session:
|
||
return _NO_BYPASS
|
||
return frozenset(per_query or ()) | frozenset(per_session or ())
|
||
|
||
|
||
def _should_attach_soft_delete_criteria(execute_state: ORMExecuteState) -> bool:
|
||
"""The event classes where the listener attaches loader criteria.
|
||
|
||
Column loads are excluded: they re-execute against already-loaded
|
||
parents to refresh specific attribute values; the soft-delete
|
||
criteria isn't meaningful at that level and would add noise.
|
||
|
||
Relationship loads ARE included. The Bayer canonical pattern
|
||
excludes them on the assumption that
|
||
``with_loader_criteria(..., propagate_to_loaders=True)`` carries
|
||
the criteria from the parent statement to its relationship loads
|
||
automatically. In practice this propagation isn't reliable when
|
||
the criteria targets a class that doesn't appear in the parent
|
||
statement (e.g., loading a ``Dashboard`` whose listener-attached
|
||
criteria targets ``Slice`` — Slice never appears in the parent
|
||
query, and the criteria doesn't always reach the
|
||
``dashboard.slices`` lazy load). Re-attaching on the
|
||
relationship-load event closes that gap. The resulting WHERE
|
||
clause may have ``deleted_at IS NULL`` twice when propagation
|
||
DOES work — harmless redundancy, idempotent SQL.
|
||
|
||
Gated by the temporary ``SOFT_DELETE`` rollout flag: while it is
|
||
off, no criteria are attached, so soft-deleted rows (which the delete path
|
||
does not create while the flag is off) are not filtered. The flag and this
|
||
gate are removed once soft delete is stable.
|
||
"""
|
||
return (
|
||
execute_state.is_select
|
||
and not execute_state.is_column_load
|
||
and is_feature_enabled("SOFT_DELETE")
|
||
)
|
||
|
||
|
||
def _all_soft_delete_subclasses() -> list[type[SoftDeleteMixin]]:
|
||
"""The cached subclass registry maintained by
|
||
``SoftDeleteMixin.__init_subclass__``. Returned in a stable
|
||
qualname-sorted order so SQLAlchemy's compiled-statement cache key
|
||
is deterministic across processes.
|
||
|
||
Assumes all soft-deletable models have been imported by the time
|
||
the listener fires. Superset imports models eagerly at app init via
|
||
``superset.models``; if that ever changes to lazy import, the
|
||
listener would silently stop filtering un-imported classes — but
|
||
since registration happens at class-definition time, the cache is
|
||
automatically updated as new subclasses are introduced (including
|
||
test-defined synthetic subclasses).
|
||
"""
|
||
return SoftDeleteMixin._registered_subclasses
|
||
|
||
|
||
def _add_soft_delete_filter(execute_state: ORMExecuteState) -> None:
|
||
"""Global ``do_orm_execute`` listener that automatically excludes
|
||
soft-deleted rows from every ORM SELECT.
|
||
|
||
Uses SQLAlchemy's recommended soft-delete pattern
|
||
(``do_orm_execute`` + ``with_loader_criteria`` — see
|
||
https://github.com/sqlalchemy/sqlalchemy/issues/7973#issuecomment-1112561295).
|
||
|
||
Which statements get the criteria is decided by
|
||
``_should_attach_soft_delete_criteria`` -- and note that relationship
|
||
loads are deliberately INCLUDED (only column loads are skipped).
|
||
``propagate_to_loaders`` alone does not reliably cover lazy loads and
|
||
can leak soft-deleted rows through them; that predicate's own docstring
|
||
explains why at length. Defer to it.
|
||
|
||
Per-class scoping: the listener iterates concrete ``SoftDeleteMixin``
|
||
subclasses and attaches a ``with_loader_criteria`` only for those
|
||
NOT in the request's bypass set. A bypass for ``Dashboard`` therefore
|
||
does not unhide soft-deleted ``Slice`` or ``SqlaTable`` rows in the
|
||
same statement. Each criteria is a concrete SQL expression rather
|
||
than a callable, so SQLAlchemy compiles it normally (passing a
|
||
callable triggers ``DeferredLambdaElement`` parsing, which does not
|
||
support Python control flow like ``if cls in bypass``).
|
||
|
||
Opt-out:
|
||
|
||
* **One statement**: attach
|
||
``execution_options(_skip_visibility_filter_classes={Model})`` to a
|
||
Query, or pass ``skip_visibility_filter=True`` to ``BaseDAO``
|
||
methods (which translate the boolean into a one-class set
|
||
internally).
|
||
* **Session-scoped**: set
|
||
``session.info[SKIP_VISIBILITY_FILTER_CLASSES] = {Model, ...}`` or
|
||
use the ``skip_visibility_filter`` context manager. Survives FAB's
|
||
inner/outer query reconstruction (and any future framework that
|
||
strips per-query options).
|
||
|
||
Performance: the listener attaches one ``with_loader_criteria``
|
||
option per non-bypassed ``SoftDeleteMixin`` subclass to every
|
||
primary SELECT, including queries that don't reference any
|
||
soft-deletable entity. SQLAlchemy no-ops the criteria when the
|
||
targeted class isn't in the statement, so the cost is small per
|
||
query, but linear in the number of soft-delete classes. At ~10
|
||
entities this is still negligible on typical endpoints; profile
|
||
before adding many more.
|
||
"""
|
||
if not _should_attach_soft_delete_criteria(execute_state):
|
||
return
|
||
|
||
bypass_classes = _collect_bypass_classes(execute_state)
|
||
|
||
for cls in _all_soft_delete_subclasses():
|
||
if cls in bypass_classes:
|
||
continue
|
||
# Pass the criteria as a lambda — SQLAlchemy adapts the column
|
||
# reference to whatever alias the class wears at each occurrence
|
||
# in the statement (critical for FAB's outer/inner reconstruction
|
||
# and any other code that aliases the same model under different
|
||
# names). A concrete SQL expression — ``cls.where_not_deleted()``
|
||
# — would render as the raw ``slices.deleted_at`` even when the
|
||
# statement actually aliases ``slices AS chart``, producing
|
||
# ``Unknown column 'slices.deleted_at' in 'on clause'``. The
|
||
# lambda's body is trivial so the ``DeferredLambdaElement``
|
||
# parser handles it without issue; complex control flow inside
|
||
# the lambda is what trips the parser, not simple attribute
|
||
# access.
|
||
execute_state.statement = execute_state.statement.options(
|
||
with_loader_criteria(
|
||
cls,
|
||
lambda c: c.deleted_at.is_(None),
|
||
include_aliases=True,
|
||
)
|
||
)
|
||
|
||
|
||
@contextmanager
|
||
def skip_visibility_filter(session: Session, *classes: type) -> Iterator[None]:
|
||
"""Bypass the soft-delete listener for the given classes within this
|
||
session for the duration of the ``with`` block.
|
||
|
||
Adds ``classes`` to ``session.info[SKIP_VISIBILITY_FILTER_CLASSES]``
|
||
on entry and removes them on exit, restoring the prior state. Nesting
|
||
is safe: an inner block only removes the classes *it* added, so the
|
||
outer block's bypass remains in effect.
|
||
|
||
Usage::
|
||
|
||
with skip_visibility_filter(session, Dashboard):
|
||
return session.query(Dashboard).filter_by(uuid=u).first()
|
||
|
||
Prefer this over manually setting ``session.info`` so the cleanup is
|
||
guaranteed even on exceptions. Calling with no classes is a no-op
|
||
(the block runs with no bypass added).
|
||
"""
|
||
bypass = session.info.setdefault(SKIP_VISIBILITY_FILTER_CLASSES, set())
|
||
added = set(classes) - bypass
|
||
bypass.update(added)
|
||
try:
|
||
yield
|
||
finally:
|
||
bypass -= added
|
||
|
||
|
||
class QueryResult: # pylint: disable=too-few-public-methods
|
||
"""Object returned by the query interface"""
|
||
|
||
def __init__( # pylint: disable=too-many-arguments
|
||
self,
|
||
df: pd.DataFrame,
|
||
query: str,
|
||
duration: timedelta,
|
||
applied_template_filters: Optional[list[str]] = None,
|
||
applied_filter_columns: Optional[list[ColumnTyping]] = None,
|
||
rejected_filter_columns: Optional[list[ColumnTyping]] = None,
|
||
status: str = QueryStatus.SUCCESS,
|
||
error_message: Optional[str] = None,
|
||
errors: Optional[list[dict[str, Any]]] = None,
|
||
from_dttm: Optional[datetime] = None,
|
||
to_dttm: Optional[datetime] = None,
|
||
sql_shifted_temporal_labels: set[str] | None = None,
|
||
) -> None:
|
||
self.df = df
|
||
self.query = query
|
||
self.duration = duration
|
||
self.applied_template_filters = applied_template_filters or []
|
||
self.applied_filter_columns = applied_filter_columns or []
|
||
self.rejected_filter_columns = rejected_filter_columns or []
|
||
self.status = status
|
||
self.error_message = error_message
|
||
self.errors = errors or []
|
||
self.from_dttm = from_dttm
|
||
self.to_dttm = to_dttm
|
||
self.sql_shifted_temporal_labels = sql_shifted_temporal_labels or set()
|
||
self.sql_rowcount = len(self.df.index) if not self.df.empty else 0
|
||
|
||
|
||
class ExtraJSONMixin:
|
||
"""Mixin to add an `extra` column (JSON) and utility methods"""
|
||
|
||
extra_json = sa.Column(MediumText(), default="{}")
|
||
|
||
@property
|
||
def extra(self) -> dict[str, Any]:
|
||
try:
|
||
return json.loads(self.extra_json or "{}") or {}
|
||
except (TypeError, json.JSONDecodeError) as exc:
|
||
logger.error(
|
||
"Unable to load an extra json: %r. Leaving empty.", exc, exc_info=True
|
||
)
|
||
return {}
|
||
|
||
@extra.setter
|
||
def extra(self, extras: dict[str, Any]) -> None:
|
||
self.extra_json = json.dumps(extras)
|
||
|
||
def set_extra_json_key(self, key: str, value: Any) -> None:
|
||
extra = self.extra
|
||
extra[key] = value
|
||
self.extra_json = json.dumps(extra)
|
||
|
||
@validates("extra_json")
|
||
def ensure_extra_json_is_not_none(
|
||
self,
|
||
_: str,
|
||
value: Optional[dict[str, Any]],
|
||
**kwargs: Any,
|
||
) -> Any:
|
||
if value is None:
|
||
return "{}"
|
||
return value
|
||
|
||
|
||
class CertificationMixin:
|
||
"""Mixin to add extra certification fields"""
|
||
|
||
extra = sa.Column(sa.Text, default="{}")
|
||
|
||
def get_extra_dict(self) -> dict[str, Any]:
|
||
try:
|
||
return json.loads(self.extra)
|
||
except (TypeError, json.JSONDecodeError):
|
||
return {}
|
||
|
||
@property
|
||
def is_certified(self) -> bool:
|
||
return bool(self.get_extra_dict().get("certification"))
|
||
|
||
@property
|
||
def certified_by(self) -> Optional[str]:
|
||
return self.get_extra_dict().get("certification", {}).get("certified_by")
|
||
|
||
@property
|
||
def certification_details(self) -> Optional[str]:
|
||
return self.get_extra_dict().get("certification", {}).get("details")
|
||
|
||
@property
|
||
def warning_markdown(self) -> Optional[str]:
|
||
return self.get_extra_dict().get("warning_markdown")
|
||
|
||
|
||
def clone_model(
|
||
target: Model,
|
||
ignore: Optional[list[str]] = None,
|
||
keep_relations: Optional[list[str]] = None,
|
||
**kwargs: Any,
|
||
) -> Model:
|
||
"""
|
||
Clone a SQLAlchemy model. By default will only clone naive column attributes.
|
||
To include relationship attributes, use `keep_relations`.
|
||
"""
|
||
ignore = ignore or []
|
||
|
||
table = target.__table__
|
||
primary_keys = table.primary_key.columns.keys()
|
||
data = {
|
||
attr: getattr(target, attr)
|
||
for attr in list(table.columns.keys()) + (keep_relations or [])
|
||
if attr not in primary_keys and attr not in ignore
|
||
}
|
||
data.update(kwargs)
|
||
|
||
return target.__class__(**data)
|
||
|
||
|
||
# todo(hugh): centralize where this code lives
|
||
class QueryStringExtended(NamedTuple):
|
||
applied_template_filters: Optional[list[str]]
|
||
applied_filter_columns: list[ColumnTyping]
|
||
rejected_filter_columns: list[ColumnTyping]
|
||
labels_expected: list[str]
|
||
prequeries: list[str]
|
||
sql: str
|
||
sql_shifted_temporal_labels: set[str]
|
||
|
||
|
||
class SqlaQuery(NamedTuple):
|
||
applied_template_filters: list[str]
|
||
applied_filter_columns: list[ColumnTyping]
|
||
rejected_filter_columns: list[ColumnTyping]
|
||
cte: Optional[str]
|
||
extra_cache_keys: list[Any]
|
||
labels_expected: list[str]
|
||
prequeries: list[str]
|
||
sqla_query: Select
|
||
sql_shifted_temporal_labels: set[str]
|
||
|
||
|
||
class ExploreMixin: # pylint: disable=too-many-public-methods
|
||
"""
|
||
Allows any flask_appbuilder.Model (Query, Table, etc.)
|
||
to be used to power a chart inside /explore
|
||
"""
|
||
|
||
sqla_aggregations = {
|
||
"COUNT_DISTINCT": lambda column_name: sa.func.COUNT(sa.distinct(column_name)),
|
||
"COUNT": sa.func.COUNT,
|
||
"SUM": sa.func.SUM,
|
||
"AVG": sa.func.AVG,
|
||
"MIN": sa.func.MIN,
|
||
"MAX": sa.func.MAX,
|
||
}
|
||
fetch_values_predicate = None
|
||
|
||
normalize_columns = False
|
||
|
||
@property
|
||
def type(self) -> str:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def db_extra(self) -> Optional[dict[str, Any]]:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def database_id(self) -> int:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def metrics(self) -> list[Any]:
|
||
return []
|
||
|
||
@property
|
||
def uid(self) -> str:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def is_rls_supported(self) -> bool:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def cache_timeout(self) -> int | None:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def column_names(self) -> list[str]:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def offset(self) -> int:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def main_dttm_col(self) -> Optional[str]:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def always_filter_main_dttm(self) -> Optional[bool]:
|
||
return False
|
||
|
||
@property
|
||
def dttm_cols(self) -> list[str]:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def db_engine_spec(self) -> builtins.type["BaseEngineSpec"]:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def database(self) -> Database:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def catalog(self) -> str:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def schema(self) -> str:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def sql(self) -> str:
|
||
raise NotImplementedError()
|
||
|
||
@property
|
||
def columns(self) -> list[Any]:
|
||
raise NotImplementedError()
|
||
|
||
def get_extra_cache_keys(self, query_obj: QueryObjectDict) -> list[Hashable]:
|
||
raise NotImplementedError()
|
||
|
||
def get_template_processor(self, **kwargs: Any) -> BaseTemplateProcessor:
|
||
raise NotImplementedError()
|
||
|
||
def get_dataset_timezone(self) -> str | None:
|
||
"""
|
||
Get the timezone configured for this dataset from the extra JSON field.
|
||
|
||
Returns an IANA timezone name (e.g., "Europe/Berlin", "America/New_York")
|
||
or None if not configured.
|
||
|
||
``extra`` is arbitrary user-supplied JSON, so the ``timezone`` key could
|
||
hold a non-string value (a number, object, list, ...). Only a string is
|
||
ever a valid IANA name, so anything else is treated as "not configured"
|
||
rather than propagating a bad value on to ``ZoneInfo``.
|
||
|
||
``extra_dict`` is provided by concrete datasources (e.g. ``SqlaTable``)
|
||
rather than this mixin, so read it defensively: subclasses without it
|
||
simply have no configured timezone.
|
||
"""
|
||
extra = getattr(self, "extra_dict", None) or {}
|
||
dataset_timezone = extra.get("timezone")
|
||
return dataset_timezone if isinstance(dataset_timezone, str) else None
|
||
|
||
def get_fetch_values_predicate(
|
||
self,
|
||
template_processor: Optional[ # pylint: disable=unused-argument
|
||
BaseTemplateProcessor
|
||
] = None,
|
||
) -> TextClause:
|
||
return self.fetch_values_predicate
|
||
|
||
def get_sqla_row_level_filters(
|
||
self,
|
||
template_processor: Optional[BaseTemplateProcessor] = None, # pylint: disable=unused-argument
|
||
include_global_guest_rls: bool = True, # pylint: disable=unused-argument
|
||
) -> list[TextClause]:
|
||
# TODO: We should refactor this mixin and remove this method
|
||
# as it exists in the BaseDatasource and is not applicable
|
||
# for datasources of type query
|
||
return []
|
||
|
||
def _resolve_denylist_schema(self, sql: str) -> Optional[str]:
|
||
"""
|
||
Resolve the effective default schema for ``DISALLOWED_SQL_TABLES``
|
||
checks, memoized per datasource instance.
|
||
|
||
An explicit datasource schema always wins. Otherwise the schema is
|
||
resolved through the query-aware ``get_default_schema_for_query`` (not
|
||
the static ``get_default_schema``) so the value matches what the engine
|
||
uses at runtime -- honoring dynamic-schema engines and URI/connect-arg
|
||
schema rules -- exactly like the SQL Lab / executor denylist gate.
|
||
|
||
The result is cached on the instance so adhoc-expression validation,
|
||
which runs once per selected column/metric/order-by, resolves the schema
|
||
at most once instead of issuing a fallback inspector round-trip per
|
||
expression.
|
||
"""
|
||
if self.schema:
|
||
return self.schema
|
||
if not hasattr(self, "_denylist_default_schema"):
|
||
catalog = self.catalog or self.database.get_default_catalog()
|
||
self._denylist_default_schema = self.database.resolve_query_default_schema(
|
||
sql, None, catalog
|
||
)
|
||
return self._denylist_default_schema
|
||
|
||
def _process_sql_expression( # pylint: disable=too-many-arguments
|
||
self,
|
||
expression: Optional[str],
|
||
database_id: int,
|
||
engine: str,
|
||
schema: str,
|
||
template_processor: Optional[BaseTemplateProcessor],
|
||
) -> Optional[str]:
|
||
if template_processor and expression:
|
||
expression = template_processor.process_template(expression)
|
||
if expression:
|
||
expression = validate_adhoc_subquery(
|
||
expression,
|
||
self.database,
|
||
self.catalog,
|
||
schema,
|
||
engine,
|
||
)
|
||
try:
|
||
expression = sanitize_clause(expression, engine)
|
||
except QueryClauseValidationException as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
# Adhoc expressions are user-controlled SQL that ends up inside a
|
||
# `literal_column(...)`. Apply the operator-configured
|
||
# `DISALLOWED_SQL_FUNCTIONS` / `DISALLOWED_SQL_TABLES` gates at the
|
||
# validation step so a dangerous function call (e.g. `version()`,
|
||
# `pg_read_file(...)`, `query_to_xml(...)`) is rejected before the
|
||
# expression is incorporated into the final SQL. This complements
|
||
# the same gate applied at query-execution time and gives the
|
||
# adhoc-expression path defense in depth.
|
||
disallowed_functions = app.config["DISALLOWED_SQL_FUNCTIONS"].get(
|
||
engine, set()
|
||
)
|
||
disallowed_tables = app.config["DISALLOWED_SQL_TABLES"].get(engine, set())
|
||
if disallowed_functions or disallowed_tables:
|
||
# `_process_select_expression` (and siblings) pre-wraps the
|
||
# input with `SELECT ...`; other callers pass bare
|
||
# expressions. Detect and don't double-wrap, otherwise
|
||
# `SELECT SELECT ...` fails the sqlglot parse.
|
||
sql_to_check = (
|
||
expression
|
||
if expression.strip().upper().startswith("SELECT")
|
||
else f"SELECT {expression}"
|
||
)
|
||
parsed = SQLScript(sql_to_check, engine=engine)
|
||
if disallowed_functions and parsed.check_functions_present(
|
||
disallowed_functions
|
||
):
|
||
raise SupersetDisallowedSQLFunctionException(disallowed_functions)
|
||
if disallowed_tables:
|
||
# Report only the tables actually found in the expression,
|
||
# mirroring the canonical execution-time gate in
|
||
# `superset.sql_lab._validate_query` so the user-facing
|
||
# error doesn't echo the operator's full denylist. Honors
|
||
# schema-qualified denylist entries (e.g.
|
||
# ``information_schema.tables``) and resolves unqualified
|
||
# references against the same runtime schema the SQL Lab /
|
||
# executor gate resolves them to (via the query-aware
|
||
# resolver, not the static inspector default), so both
|
||
# surfaces enforce the denylist consistently.
|
||
effective_schema = self._resolve_denylist_schema(sql_to_check)
|
||
found_tables = parsed.get_disallowed_tables(
|
||
disallowed_tables, effective_schema
|
||
)
|
||
if found_tables:
|
||
raise SupersetDisallowedSQLTableException(found_tables)
|
||
return expression
|
||
|
||
def _process_select_expression(
|
||
self,
|
||
expression: Optional[str],
|
||
database_id: int,
|
||
engine: str,
|
||
schema: str,
|
||
template_processor: Optional[BaseTemplateProcessor],
|
||
) -> Optional[str]:
|
||
"""
|
||
Validate and process an adhoc expression used as a column or metric.
|
||
|
||
This requires prefixing the expression with a dummy SELECT statement, so it can
|
||
be properly parsed and validated.
|
||
"""
|
||
if expression:
|
||
expression = f"SELECT {expression}"
|
||
|
||
if processed := self._process_sql_expression(
|
||
expression=expression,
|
||
database_id=database_id,
|
||
engine=engine,
|
||
schema=schema,
|
||
template_processor=template_processor,
|
||
):
|
||
prefix, expression = re.split(
|
||
r"SELECT\s+",
|
||
processed,
|
||
maxsplit=1,
|
||
flags=re.IGNORECASE,
|
||
)
|
||
return expression.strip()
|
||
|
||
return None
|
||
|
||
def _process_orderby_expression(
|
||
self,
|
||
expression: Optional[str],
|
||
database_id: int,
|
||
engine: str,
|
||
schema: str,
|
||
template_processor: Optional[BaseTemplateProcessor],
|
||
) -> Optional[str]:
|
||
"""
|
||
Validate and process an ORDER BY clause expression.
|
||
|
||
This requires prefixing the expression with a dummy SELECT statement, so it can
|
||
be properly parsed and validated.
|
||
"""
|
||
if expression:
|
||
expression = f"SELECT 1 ORDER BY {expression}"
|
||
|
||
if processed := self._process_sql_expression(
|
||
expression=expression,
|
||
database_id=database_id,
|
||
engine=engine,
|
||
schema=schema,
|
||
template_processor=template_processor,
|
||
):
|
||
prefix, expression = re.split(
|
||
r"ORDER\s+BY",
|
||
processed,
|
||
maxsplit=1,
|
||
flags=re.IGNORECASE,
|
||
)
|
||
return expression.strip()
|
||
|
||
return None
|
||
|
||
def make_sqla_column_compatible(
|
||
self, sqla_col: ColumnElement, label: Optional[str] = None
|
||
) -> ColumnElement:
|
||
"""Takes a sqlalchemy column object and adds label info if supported by engine.
|
||
:param sqla_col: sqlalchemy column instance
|
||
:param label: alias/label that column is expected to have
|
||
:return: either a sql alchemy column or label instance if supported by engine
|
||
"""
|
||
label_expected = label or sqla_col.name
|
||
db_engine_spec = self.db_engine_spec
|
||
# add quotes to tables
|
||
if db_engine_spec.get_allows_alias_in_select(self.database):
|
||
label = db_engine_spec.make_label_compatible(label_expected)
|
||
sqla_col = sqla_col.label(label)
|
||
sqla_col.key = label_expected
|
||
return sqla_col
|
||
|
||
@staticmethod
|
||
def _apply_cte(sql: str, cte: Optional[str]) -> str:
|
||
"""
|
||
Append a CTE before the SELECT statement if defined
|
||
|
||
:param sql: SELECT statement
|
||
:param cte: CTE statement
|
||
:return:
|
||
"""
|
||
if cte:
|
||
sql = f"{cte}\n{sql}"
|
||
return sql
|
||
|
||
def get_query_str_extended(
|
||
self,
|
||
query_obj: QueryObjectDict,
|
||
mutate: bool = True,
|
||
) -> QueryStringExtended:
|
||
# Filter out keys that aren't parameters to get_sqla_query
|
||
filtered_query_obj = {
|
||
k: v for k, v in query_obj.items() if k in SQLA_QUERY_KEYS
|
||
}
|
||
sqlaq = self.get_sqla_query(**cast(Any, filtered_query_obj))
|
||
sql = self.database.compile_sqla_query(
|
||
sqlaq.sqla_query,
|
||
catalog=self.catalog,
|
||
schema=self.schema,
|
||
is_virtual=bool(self.sql),
|
||
)
|
||
sql = self._apply_cte(sql, sqlaq.cte)
|
||
|
||
if mutate:
|
||
sql = self.database.mutate_sql_based_on_config(sql)
|
||
return QueryStringExtended(
|
||
applied_template_filters=sqlaq.applied_template_filters,
|
||
applied_filter_columns=sqlaq.applied_filter_columns,
|
||
rejected_filter_columns=sqlaq.rejected_filter_columns,
|
||
labels_expected=sqlaq.labels_expected,
|
||
prequeries=sqlaq.prequeries,
|
||
sql=sql,
|
||
sql_shifted_temporal_labels=sqlaq.sql_shifted_temporal_labels,
|
||
)
|
||
|
||
def _normalize_prequery_result_type(
|
||
self,
|
||
row: pd.Series,
|
||
dimension: str,
|
||
columns_by_name: dict[str, "TableColumn"],
|
||
) -> Union[str, int, float, bool, str]:
|
||
"""
|
||
Convert a prequery result type to its equivalent Python type.
|
||
|
||
Some databases like Druid will return timestamps as strings, but do not perform
|
||
automatic casting when comparing these strings to a timestamp. For cases like
|
||
this we convert the value via the appropriate SQL transform.
|
||
|
||
:param row: A prequery record
|
||
:param dimension: The dimension name
|
||
:param columns_by_name: The mapping of columns by name
|
||
:return: equivalent primitive python type
|
||
"""
|
||
|
||
value = row[dimension]
|
||
|
||
if isinstance(value, np.generic):
|
||
value = value.item()
|
||
|
||
column_ = columns_by_name.get(dimension)
|
||
db_extra: dict[str, Any] = self.database.get_extra()
|
||
|
||
if column_ is None:
|
||
# Column not found, return value as-is
|
||
pass
|
||
elif isinstance(column_, dict):
|
||
if (
|
||
column_.get("type")
|
||
and column_.get("is_temporal")
|
||
and isinstance(value, str)
|
||
):
|
||
sql = self.db_engine_spec.convert_dttm(
|
||
column_.get("type"), dateutil.parser.parse(value), db_extra=None
|
||
)
|
||
|
||
if sql:
|
||
value = self.db_engine_spec.get_text_clause(sql)
|
||
else:
|
||
if column_.type and column_.is_temporal and isinstance(value, str):
|
||
sql = self.db_engine_spec.convert_dttm(
|
||
column_.type, dateutil.parser.parse(value), db_extra=db_extra
|
||
)
|
||
|
||
if sql:
|
||
value = self.db_engine_spec.get_text_clause(sql)
|
||
return value
|
||
|
||
def make_orderby_compatible(
|
||
self, select_exprs: list[ColumnElement], orderby_exprs: list[ColumnElement]
|
||
) -> None:
|
||
"""
|
||
If needed, make sure aliases for selected columns are not used in
|
||
`ORDER BY`.
|
||
|
||
In some databases (e.g. Presto), `ORDER BY` clause is not able to
|
||
automatically pick the source column if a `SELECT` clause alias is named
|
||
the same as a source column. In this case, we update the SELECT alias to
|
||
another name to avoid the conflict.
|
||
"""
|
||
if self.db_engine_spec.allows_alias_to_source_column:
|
||
return
|
||
|
||
def is_alias_used_in_orderby(col: ColumnElement) -> bool:
|
||
if not isinstance(col, Label):
|
||
return False
|
||
regexp = re.compile(f"\\(.*\\b{re.escape(col.name)}\\b.*\\)", re.IGNORECASE)
|
||
return any(regexp.search(str(x)) for x in orderby_exprs)
|
||
|
||
# Iterate through selected columns, if column alias appears in orderby
|
||
# use another `alias`. The final output columns will still use the
|
||
# original names, because they are updated by `labels_expected` after
|
||
# querying.
|
||
for col in select_exprs:
|
||
if is_alias_used_in_orderby(col):
|
||
col.name = f"{col.name}__"
|
||
|
||
def _raise_for_disallowed_sql(self, sql: str) -> None:
|
||
"""
|
||
Mirror the DISALLOWED_SQL_* gate that sql_lab.execute_sql_statement
|
||
enforces so both query surfaces honour the same denylist.
|
||
"""
|
||
engine = self.db_engine_spec.engine
|
||
disallowed_functions = app.config["DISALLOWED_SQL_FUNCTIONS"].get(engine, set())
|
||
disallowed_tables = app.config["DISALLOWED_SQL_TABLES"].get(engine, set())
|
||
if not (disallowed_functions or disallowed_tables):
|
||
return
|
||
|
||
parsed_script = SQLScript(sql, engine=engine)
|
||
if disallowed_functions and parsed_script.check_functions_present(
|
||
disallowed_functions
|
||
):
|
||
raise SupersetDisallowedSQLFunctionException(disallowed_functions)
|
||
if disallowed_tables:
|
||
# Report only the tables actually found in the query, mirroring the
|
||
# canonical execution-time gate so the user-facing error doesn't
|
||
# echo the operator's full denylist. Honors schema-qualified
|
||
# denylist entries (e.g. ``information_schema.tables``) and resolves
|
||
# unqualified references against the same runtime schema the SQL Lab
|
||
# / executor gate resolves them to (via the query-aware resolver,
|
||
# not the static inspector default), so both surfaces enforce the
|
||
# denylist consistently.
|
||
effective_schema = self._resolve_denylist_schema(sql)
|
||
found_tables = parsed_script.get_disallowed_tables(
|
||
disallowed_tables, effective_schema
|
||
)
|
||
if found_tables:
|
||
raise SupersetDisallowedSQLTableException(found_tables)
|
||
|
||
def query(self, query_obj: QueryObjectDict) -> QueryResult:
|
||
"""
|
||
Executes the query and returns a dataframe.
|
||
|
||
This method is the unified entry point for query execution across all
|
||
datasource types (Query, SqlaTable, etc.).
|
||
"""
|
||
qry_start_dttm = datetime.now()
|
||
query_str_ext = self.get_query_str_extended(query_obj)
|
||
sql = query_str_ext.sql
|
||
|
||
self._raise_for_disallowed_sql(sql)
|
||
|
||
status = QueryStatus.SUCCESS
|
||
errors = None
|
||
error_message = None
|
||
|
||
def assign_column_label(df: pd.DataFrame) -> Optional[pd.DataFrame]:
|
||
"""
|
||
Some engines change the case or generate bespoke column names, either by
|
||
default or due to lack of support for aliasing. This function ensures that
|
||
the column names in the DataFrame correspond to what is expected by
|
||
the viz components.
|
||
Sometimes a query may also contain only order by columns that are not used
|
||
as metrics or groupby columns, but need to present in the SQL `select`,
|
||
filtering by `labels_expected` make sure we only return columns users want.
|
||
:param df: Original DataFrame returned by the engine
|
||
:return: Mutated DataFrame
|
||
"""
|
||
labels_expected = query_str_ext.labels_expected
|
||
if df is not None and not df.empty:
|
||
if len(df.columns) < len(labels_expected):
|
||
raise QueryObjectValidationError(
|
||
_("Db engine did not return all queried columns")
|
||
)
|
||
if len(df.columns) > len(labels_expected):
|
||
df = df.iloc[:, 0 : len(labels_expected)]
|
||
df.columns = labels_expected
|
||
return df
|
||
|
||
extras = query_obj.get("extras") or {}
|
||
system_sampling = bool(extras.get("system_sampling")) and not self.sql
|
||
|
||
def run_query(query_sql: str) -> Optional[pd.DataFrame]:
|
||
return self.database.get_df(
|
||
query_sql,
|
||
self.catalog,
|
||
self.schema,
|
||
mutator=assign_column_label,
|
||
)
|
||
|
||
try:
|
||
if system_sampling:
|
||
# System-authored sampling over a physical table (e.g. the
|
||
# Samples tab): when the engine rejects the statement with a
|
||
# read-limit error, retry once with the engine's bounded-read
|
||
# override so a partial sample is returned instead of an error.
|
||
df = self.database.run_with_sampling_read_limit_retry(sql, run_query)
|
||
else:
|
||
df = run_query(sql)
|
||
except Exception as ex: # pylint: disable=broad-except
|
||
# Re-raise SupersetErrorException (includes OAuth2RedirectError)
|
||
# to bubble up to API layer
|
||
if isinstance(ex, (SupersetErrorException, SupersetErrorsException)):
|
||
raise
|
||
df = pd.DataFrame()
|
||
status = QueryStatus.FAILED
|
||
logger.warning(
|
||
"Query %s on schema %s failed", sql, self.schema, exc_info=True
|
||
)
|
||
db_engine_spec = self.db_engine_spec
|
||
errors = [
|
||
dataclasses.asdict(error)
|
||
for error in db_engine_spec.extract_errors(
|
||
ex, database_name=self.database.unique_name
|
||
)
|
||
]
|
||
error_message = utils.error_msg_from_exception(ex)
|
||
|
||
return QueryResult(
|
||
applied_template_filters=query_str_ext.applied_template_filters,
|
||
applied_filter_columns=query_str_ext.applied_filter_columns,
|
||
rejected_filter_columns=query_str_ext.rejected_filter_columns,
|
||
status=status,
|
||
df=df,
|
||
duration=datetime.now() - qry_start_dttm,
|
||
query=sql,
|
||
errors=errors,
|
||
error_message=error_message,
|
||
sql_shifted_temporal_labels=query_str_ext.sql_shifted_temporal_labels,
|
||
)
|
||
|
||
def exc_query(self, qry: Any) -> QueryResult:
|
||
"""
|
||
Deprecated: Use query() instead.
|
||
This method is kept for backward compatibility.
|
||
"""
|
||
return self.query(qry)
|
||
|
||
def _python_date_format(self, column: str | None) -> str | None:
|
||
"""Return the column's configured ``python_date_format`` (e.g. ``epoch_s``
|
||
or a strftime pattern), or ``None`` if the column declares no format.
|
||
Reads from either a column object or a dict, matching ``_is_dttm``."""
|
||
if not hasattr(self, "get_column") or not (col := self.get_column(column)):
|
||
return None
|
||
fmt = (
|
||
col.get("python_date_format")
|
||
if isinstance(col, dict)
|
||
else getattr(col, "python_date_format", None)
|
||
)
|
||
return str(fmt) if fmt else None
|
||
|
||
def _collect_dttm_labels(
|
||
self, query_object: QueryObject
|
||
) -> tuple[tuple[str, str | None], ...]:
|
||
"""``(label, python_date_format)`` for the columns whose values should be
|
||
normalized to datetimes: base-axis / granularity columns (aggregated
|
||
charts), plus raw/unaggregated temporal columns that declare a
|
||
``python_date_format``. The raw columns are gated on the declared format
|
||
rather than ``is_dttm`` alone so a plain integer column is not misread as
|
||
nanosecond timestamps. The format is resolved here with a single column
|
||
lookup per label so callers need not look it up again."""
|
||
|
||
def _resolve(label: str | None) -> tuple[bool, str | None]:
|
||
"""``(is_dttm, python_date_format)`` from one ``get_column`` lookup."""
|
||
if not hasattr(self, "get_column") or not (col := self.get_column(label)):
|
||
return False, None
|
||
if isinstance(col, dict):
|
||
fmt = col.get("python_date_format")
|
||
return bool(col.get("is_dttm")), str(fmt) if fmt else None
|
||
fmt = getattr(col, "python_date_format", None)
|
||
return bool(col.is_dttm), str(fmt) if fmt else None
|
||
|
||
labels: list[tuple[str, str | None]] = []
|
||
seen: set[str] = set()
|
||
for label in [
|
||
*get_base_axis_labels(query_object.columns),
|
||
query_object.granularity,
|
||
]:
|
||
if not label or label in seen:
|
||
continue
|
||
is_dttm, fmt = _resolve(label)
|
||
if is_dttm:
|
||
labels.append((label, fmt))
|
||
seen.add(label)
|
||
for label in get_column_names(query_object.columns):
|
||
if label in seen:
|
||
continue
|
||
is_dttm, fmt = _resolve(label)
|
||
if is_dttm and fmt:
|
||
labels.append((label, fmt))
|
||
seen.add(label)
|
||
return tuple(labels)
|
||
|
||
def _offset_only_dttm_cols(
|
||
self,
|
||
df: pd.DataFrame,
|
||
query_object: QueryObject,
|
||
already_collected: set[str],
|
||
sql_shifted_temporal_labels: set[str] | None = None,
|
||
) -> list[DateColumn]:
|
||
"""``DateColumn`` entries that only need the dataset HOURS OFFSET (and any
|
||
time shift) applied, for temporal columns the database already returns as
|
||
native datetimes.
|
||
|
||
The dataset offset must apply to *every* temporal column a query returns,
|
||
not just the selected time column. ``_collect_dttm_labels`` only covers
|
||
columns that need parsing (a declared ``python_date_format``) or the
|
||
base-axis / granularity column; a second temporal column returned as a
|
||
native datetime would otherwise keep its raw, un-offset value. Columns
|
||
arriving as plain integers/strings without a declared format are skipped,
|
||
since they cannot be safely interpreted as datetimes.
|
||
See https://github.com/apache/superset/issues/23167.
|
||
"""
|
||
if not (self.offset or query_object.time_shift) or not hasattr(
|
||
self, "get_column"
|
||
):
|
||
return []
|
||
|
||
sql_shifted_temporal_labels = sql_shifted_temporal_labels or set()
|
||
extra: list[DateColumn] = []
|
||
for label in df.columns:
|
||
if label in already_collected or label == DTTM_ALIAS:
|
||
continue
|
||
if not pd.api.types.is_datetime64_any_dtype(df[label]):
|
||
continue
|
||
column_obj = self.get_column(label)
|
||
if not column_obj:
|
||
continue
|
||
is_dttm = (
|
||
column_obj.get("is_dttm")
|
||
if isinstance(column_obj, dict)
|
||
else getattr(column_obj, "is_dttm", False)
|
||
)
|
||
if is_dttm:
|
||
extra.append(
|
||
DateColumn(
|
||
timestamp_format=None,
|
||
offset=(
|
||
0 if label in sql_shifted_temporal_labels else self.offset
|
||
),
|
||
time_shift=query_object.time_shift,
|
||
col_label=label,
|
||
)
|
||
)
|
||
already_collected.add(label)
|
||
return extra
|
||
|
||
def normalize_df(
|
||
self,
|
||
df: pd.DataFrame,
|
||
query_object: QueryObject,
|
||
sql_shifted_temporal_labels: set[str] | None = None,
|
||
) -> pd.DataFrame:
|
||
"""
|
||
Normalize the dataframe by converting datetime columns and ensuring
|
||
numerical metrics.
|
||
|
||
:param df: The dataframe to normalize
|
||
:param query_object: The query object with metadata about columns
|
||
:param sql_shifted_temporal_labels: labels already shifted in generated SQL
|
||
:return: Normalized dataframe
|
||
"""
|
||
sql_shifted_temporal_labels = sql_shifted_temporal_labels or set()
|
||
labels = self._collect_dttm_labels(query_object)
|
||
|
||
# ``get_dataset_timezone`` lives on ``ExploreMixin``; datasource doubles
|
||
# that bind only a subset of mixin methods onto a plain object (as some
|
||
# unit tests do) won't have it, so fall back to "not configured" rather
|
||
# than raising.
|
||
get_dataset_timezone = getattr(self, "get_dataset_timezone", None)
|
||
dataset_timezone = get_dataset_timezone() if get_dataset_timezone else None
|
||
dttm_cols = [
|
||
DateColumn(
|
||
timestamp_format=fmt,
|
||
offset=0 if label in sql_shifted_temporal_labels else self.offset,
|
||
time_shift=query_object.time_shift,
|
||
timezone=dataset_timezone,
|
||
col_label=label,
|
||
)
|
||
for label, fmt in labels
|
||
]
|
||
|
||
if DTTM_ALIAS in df:
|
||
dttm_cols.append(
|
||
DateColumn.get_legacy_time_column(
|
||
timestamp_format=self._python_date_format(query_object.granularity),
|
||
offset=(
|
||
0 if DTTM_ALIAS in sql_shifted_temporal_labels else self.offset
|
||
),
|
||
time_shift=query_object.time_shift,
|
||
timezone=dataset_timezone,
|
||
)
|
||
)
|
||
|
||
dttm_cols.extend(
|
||
self._offset_only_dttm_cols(
|
||
df,
|
||
query_object,
|
||
{col.col_label for col in dttm_cols},
|
||
sql_shifted_temporal_labels,
|
||
)
|
||
)
|
||
|
||
# Build format map from detected datetime formats stored in dataset columns
|
||
format_map: dict[str, str] = {}
|
||
if hasattr(self, "columns"):
|
||
for col in self.columns:
|
||
if hasattr(col, "datetime_format") and col.datetime_format:
|
||
format_map[col.column_name] = col.datetime_format
|
||
|
||
normalize_dttm_col(
|
||
df=df,
|
||
dttm_cols=tuple(dttm_cols),
|
||
format_map=format_map if format_map else None,
|
||
)
|
||
|
||
# Convert metrics to numerical values if enforced
|
||
if getattr(self, "enforce_numerical_metrics", True):
|
||
dataframe_utils.df_metrics_to_num(df, query_object)
|
||
|
||
df.replace([np.inf, -np.inf], np.nan, inplace=True)
|
||
|
||
return df
|
||
|
||
def get_query_result(self, query_object: QueryObject) -> QueryResult:
|
||
"""
|
||
Execute query and return results with full processing pipeline.
|
||
|
||
This method handles:
|
||
1. Query execution via self.query()
|
||
2. DataFrame normalization
|
||
3. Time offset processing (if applicable)
|
||
4. Post-processing operations
|
||
|
||
:param query_object: The query configuration
|
||
:return: QueryResult with processed dataframe
|
||
"""
|
||
# Execute the base query
|
||
result = self.query(query_object.to_dict())
|
||
query = result.query + ";\n\n" if result.query else ""
|
||
|
||
# Process the dataframe if not empty
|
||
df = result.df
|
||
if not df.empty:
|
||
# Normalize datetime columns and metrics
|
||
df = self.normalize_df(
|
||
df,
|
||
query_object,
|
||
result.sql_shifted_temporal_labels,
|
||
)
|
||
|
||
# Process time offsets if requested
|
||
if query_object.time_offsets:
|
||
# Process time offsets using the datasource's own method
|
||
# Note: caching is disabled here as we don't have query context
|
||
time_offsets = self.processing_time_offsets(
|
||
df, query_object, cache_key_fn=None, cache_timeout_fn=None
|
||
)
|
||
df = time_offsets["df"]
|
||
queries = time_offsets["queries"]
|
||
query += ";\n\n".join(queries)
|
||
query += ";\n\n"
|
||
|
||
# Execute post-processing operations
|
||
try:
|
||
df = query_object.exec_post_processing(df)
|
||
except InvalidPostProcessingError as ex:
|
||
raise QueryObjectValidationError(ex.message) from ex
|
||
|
||
# Update result with processed data
|
||
result.df = df
|
||
result.query = query
|
||
result.from_dttm = query_object.from_dttm
|
||
result.to_dttm = query_object.to_dttm
|
||
|
||
return result
|
||
|
||
def processing_time_offsets( # pylint: disable=too-many-locals,too-many-statements # noqa: C901
|
||
self,
|
||
df: pd.DataFrame,
|
||
query_object: QueryObject,
|
||
cache_key_fn: Callable[[QueryObject, str, Any], str | None] | None = None,
|
||
cache_timeout_fn: Callable[[], int] | None = None,
|
||
force_cache: bool = False,
|
||
) -> CachedTimeOffset:
|
||
"""
|
||
Process time offsets for time comparison feature.
|
||
|
||
This method handles both relative time offsets (e.g., "1 week ago") and
|
||
absolute date range offsets (e.g., "2015-01-03 : 2015-01-04").
|
||
|
||
:param df: The main dataframe
|
||
:param query_object: The query object with time offset configuration
|
||
:param cache_key_fn: Optional function to generate cache keys
|
||
:param cache_timeout_fn: Optional function to get cache timeout
|
||
:param force_cache: Whether to force cache refresh
|
||
:return: CachedTimeOffset with processed dataframe and queries
|
||
"""
|
||
# Import here to avoid circular dependency
|
||
# pylint: disable=import-outside-toplevel
|
||
from superset.common.utils.query_cache_manager import QueryCacheManager
|
||
|
||
# ensure query_object is immutable
|
||
query_object_clone = copy.copy(query_object)
|
||
queries: list[str] = []
|
||
cache_keys: list[str | None] = []
|
||
offset_dfs: dict[str, pd.DataFrame] = {}
|
||
|
||
outer_from_dttm, outer_to_dttm = get_since_until_from_query_object(query_object)
|
||
if not outer_from_dttm or not outer_to_dttm:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"An enclosed time range (both start and end) must be specified "
|
||
"when using a Time Comparison."
|
||
)
|
||
)
|
||
|
||
time_grain = self.get_time_grain(query_object)
|
||
metric_names = get_metric_names(query_object.metrics)
|
||
# use columns that are not metrics as join keys
|
||
join_keys = [col for col in df.columns if col not in metric_names]
|
||
|
||
for offset in query_object.time_offsets:
|
||
try:
|
||
original_offset = offset
|
||
is_date_range_offset = self.is_valid_date_range(offset)
|
||
|
||
if is_date_range_offset and feature_flag_manager.is_feature_enabled(
|
||
"DATE_RANGE_TIMESHIFTS_ENABLED"
|
||
):
|
||
# DATE RANGE OFFSET LOGIC (like "2015-01-03 : 2015-01-04")
|
||
try:
|
||
# Parse the specified range
|
||
offset_from_dttm, offset_to_dttm = (
|
||
get_since_until_from_time_range(time_range=offset)
|
||
)
|
||
except ValueError as ex:
|
||
raise QueryObjectValidationError(str(ex)) from ex
|
||
|
||
# Use the specified range directly
|
||
query_object_clone.from_dttm = offset_from_dttm
|
||
query_object_clone.to_dttm = offset_to_dttm
|
||
|
||
# For date range offsets, we must NOT set inner bounds
|
||
# These create additional WHERE clauses that conflict with our
|
||
# date range
|
||
query_object_clone.inner_from_dttm = None
|
||
query_object_clone.inner_to_dttm = None
|
||
|
||
elif is_date_range_offset:
|
||
# Date range timeshift feature is disabled
|
||
raise QueryObjectValidationError(
|
||
"Date range timeshifts are not enabled. "
|
||
"Please contact your administrator to enable the "
|
||
"DATE_RANGE_TIMESHIFTS_ENABLED feature flag."
|
||
)
|
||
|
||
else:
|
||
# RELATIVE OFFSET LOGIC (like "1 day ago")
|
||
if self.is_valid_date(offset) or offset == "inherit":
|
||
offset = self.get_offset_custom_or_inherit(
|
||
offset,
|
||
outer_from_dttm,
|
||
outer_to_dttm,
|
||
)
|
||
elif not is_parseable_human_timedelta(offset):
|
||
# get_past_or_future silently returns the source time
|
||
# for offsets it cannot parse; querying with an
|
||
# unshifted window would present the current period's
|
||
# data as the comparison series. The parse flag (not
|
||
# a zero delta) is the unparseability signal, so
|
||
# legitimate zero-shift offsets like "0 days ago"
|
||
# pass through.
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Unable to interpret the time offset: "
|
||
"%(offset)s. Use a relative time such as "
|
||
'"1 month ago".',
|
||
offset=offset,
|
||
)
|
||
)
|
||
query_object_clone.from_dttm = get_past_or_future(
|
||
offset,
|
||
outer_from_dttm,
|
||
)
|
||
query_object_clone.to_dttm = get_past_or_future(
|
||
offset, outer_to_dttm
|
||
)
|
||
|
||
query_object_clone.inner_from_dttm = outer_from_dttm
|
||
query_object_clone.inner_to_dttm = outer_to_dttm
|
||
|
||
x_axis_label = get_x_axis_label(query_object.columns)
|
||
query_object_clone.granularity = (
|
||
query_object_clone.granularity or x_axis_label
|
||
)
|
||
|
||
except ValueError as ex:
|
||
raise QueryObjectValidationError(str(ex)) from ex
|
||
|
||
query_object_clone.time_offsets = []
|
||
query_object_clone.post_processing = []
|
||
|
||
# Get time offset index
|
||
index = (get_base_axis_labels(query_object.columns) or [DTTM_ALIAS])[0]
|
||
|
||
if is_date_range_offset and feature_flag_manager.is_feature_enabled(
|
||
"DATE_RANGE_TIMESHIFTS_ENABLED"
|
||
):
|
||
# Create a completely new filter list to preserve original filters
|
||
query_object_clone.filter = copy.deepcopy(query_object_clone.filter)
|
||
|
||
# Remove any existing temporal filters that might conflict
|
||
query_object_clone.filter = [
|
||
flt
|
||
for flt in query_object_clone.filter
|
||
if not (flt.get("op") == FilterOperator.TEMPORAL_RANGE)
|
||
]
|
||
|
||
# Determine the temporal column with multiple fallback strategies
|
||
temporal_col = self._get_temporal_column_for_filter(
|
||
query_object, x_axis_label
|
||
)
|
||
|
||
# Always add a temporal filter for date range offsets
|
||
if temporal_col:
|
||
new_temporal_filter: QueryObjectFilterClause = {
|
||
"col": temporal_col,
|
||
"op": FilterOperator.TEMPORAL_RANGE,
|
||
"val": (
|
||
f"{query_object_clone.from_dttm} : "
|
||
f"{query_object_clone.to_dttm}"
|
||
),
|
||
}
|
||
query_object_clone.filter.append(new_temporal_filter)
|
||
|
||
else:
|
||
# This should rarely happen with proper fallbacks
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Unable to identify temporal column for date range time comparison." # noqa: E501
|
||
"Please ensure your dataset has a properly configured time column." # noqa: E501
|
||
)
|
||
)
|
||
|
||
else:
|
||
# RELATIVE OFFSET: Original logic for non-date-range offsets
|
||
# The comparison is not using a temporal column so we need to modify
|
||
# the temporal filter so we run the query with the correct time range
|
||
if not dataframe_utils.is_datetime_series(df.get(index)):
|
||
query_object_clone.filter = copy.deepcopy(query_object_clone.filter)
|
||
|
||
# Find and update temporal filters
|
||
for flt in query_object_clone.filter:
|
||
if flt.get(
|
||
"op"
|
||
) == FilterOperator.TEMPORAL_RANGE and isinstance(
|
||
flt.get("val"), str
|
||
):
|
||
time_range = cast(str, flt.get("val"))
|
||
(
|
||
new_outer_from_dttm,
|
||
new_outer_to_dttm,
|
||
) = get_since_until_from_time_range(
|
||
time_range=time_range,
|
||
time_shift=offset,
|
||
)
|
||
flt["val"] = f"{new_outer_from_dttm} : {new_outer_to_dttm}"
|
||
else:
|
||
# If it IS a datetime series, we still need to clear conflicts
|
||
query_object_clone.filter = copy.deepcopy(query_object_clone.filter)
|
||
|
||
# Match against the column of the existing TEMPORAL_RANGE
|
||
# filter rather than the X-axis label so adhoc Custom SQL
|
||
# x-axes (label != underlying time column) still get shifted.
|
||
temporal_col = self._get_temporal_column_for_filter(
|
||
query_object, x_axis_label
|
||
)
|
||
|
||
# Update any existing temporal filters to match our shifted range
|
||
for flt in query_object_clone.filter:
|
||
if (
|
||
flt.get("op") == FilterOperator.TEMPORAL_RANGE
|
||
and flt.get("col") == temporal_col
|
||
):
|
||
flt["val"] = (
|
||
f"{query_object_clone.from_dttm} : "
|
||
f"{query_object_clone.to_dttm}"
|
||
)
|
||
|
||
# Remove non-temporal x-axis filters (but keep temporal ones)
|
||
query_object_clone.filter = [
|
||
flt
|
||
for flt in query_object_clone.filter
|
||
if not (
|
||
flt.get("col") == x_axis_label
|
||
and flt.get("op") != FilterOperator.TEMPORAL_RANGE
|
||
)
|
||
]
|
||
|
||
# Continue with the rest of the method (caching, execution, etc.)
|
||
cached_time_offset_key = (
|
||
offset if offset == original_offset else f"{offset}_{original_offset}"
|
||
)
|
||
|
||
cache_key = None
|
||
if cache_key_fn:
|
||
cache_key = cache_key_fn(
|
||
query_object_clone,
|
||
cached_time_offset_key,
|
||
time_grain,
|
||
)
|
||
|
||
cache = QueryCacheManager.get(cache_key, CacheRegion.DATA, force_cache)
|
||
|
||
if cache.is_loaded:
|
||
offset_dfs[offset] = cache.df
|
||
queries.append(cache.query)
|
||
cache_keys.append(cache_key)
|
||
continue
|
||
|
||
query_object_clone_dct = query_object_clone.to_dict()
|
||
|
||
# rename metrics: SUM(value) => SUM(value) 1 year ago
|
||
metrics_mapping = {
|
||
metric: TIME_COMPARISON.join([metric, original_offset])
|
||
for metric in metric_names
|
||
}
|
||
|
||
# The subquery drops row_offset (the offset period's own row ordering
|
||
# differs from the main query's, so applying the same offset would skew
|
||
# the join). It must still fetch enough rows to cover the main query's
|
||
# window, hence row_limit + row_offset when a chart limit is set.
|
||
if query_object.row_limit or query_object.row_offset:
|
||
if query_object.row_limit:
|
||
query_object_clone_dct["row_limit"] = (
|
||
query_object.row_limit + query_object.row_offset
|
||
)
|
||
else:
|
||
query_object_clone_dct["row_limit"] = app.config["ROW_LIMIT"]
|
||
query_object_clone_dct["row_offset"] = 0
|
||
|
||
# Call the unified query method on the datasource
|
||
result = self.query(query_object_clone_dct)
|
||
|
||
queries.append(result.query)
|
||
cache_keys.append(None)
|
||
|
||
offset_metrics_df = result.df
|
||
if offset_metrics_df.empty:
|
||
offset_metrics_df = pd.DataFrame(
|
||
{
|
||
col: [np.nan]
|
||
for col in join_keys + list(metrics_mapping.values())
|
||
}
|
||
)
|
||
else:
|
||
# 1. normalize df, set dttm column
|
||
offset_metrics_df = self.normalize_df(
|
||
offset_metrics_df,
|
||
query_object_clone,
|
||
result.sql_shifted_temporal_labels,
|
||
)
|
||
|
||
# 2. rename extra query columns
|
||
offset_metrics_df = offset_metrics_df.rename(columns=metrics_mapping)
|
||
|
||
# cache df and query if caching is enabled
|
||
if cache_key and cache_timeout_fn:
|
||
value = {
|
||
"df": offset_metrics_df,
|
||
"query": result.query,
|
||
}
|
||
cache.set(
|
||
key=cache_key,
|
||
value=value,
|
||
timeout=cache_timeout_fn(),
|
||
datasource_uid=self.uid,
|
||
region=CacheRegion.DATA,
|
||
)
|
||
offset_dfs[offset] = offset_metrics_df
|
||
|
||
if offset_dfs:
|
||
x_axis_columns = get_base_axis_columns(query_object.columns)
|
||
x_axis_expression = (
|
||
x_axis_columns[0].get("sqlExpression")
|
||
if x_axis_columns and isinstance(x_axis_columns[0], dict)
|
||
else None
|
||
)
|
||
x_axis_metadata = _get_temporal_physical_column_metadata(
|
||
self,
|
||
x_axis_expression if isinstance(x_axis_expression, str) else None,
|
||
)
|
||
df = self.join_offset_dfs(
|
||
df,
|
||
offset_dfs,
|
||
time_grain,
|
||
join_keys,
|
||
full_range=getattr(query_object, "time_compare_full_range", False),
|
||
x_axis_label=get_x_axis_label(query_object.columns),
|
||
x_axis_is_temporal=x_axis_metadata.is_temporal,
|
||
x_axis_datetime_format=x_axis_metadata.python_date_format,
|
||
)
|
||
|
||
return CachedTimeOffset(df=df, queries=queries, cache_keys=cache_keys)
|
||
|
||
@staticmethod
|
||
def get_time_grain(query_object: QueryObject) -> Any | None:
|
||
if (
|
||
query_object.columns
|
||
and len(query_object.columns) > 0
|
||
and isinstance(query_object.columns[0], dict)
|
||
):
|
||
# If the time grain is in the columns it will be the first one
|
||
# and it will be of AdhocColumn type
|
||
return query_object.columns[0].get("timeGrain")
|
||
|
||
return query_object.extras.get("time_grain_sqla")
|
||
|
||
def is_valid_date(self, date_string: str) -> bool:
|
||
try:
|
||
# Attempt to parse the string as a date in the format YYYY-MM-DD
|
||
datetime.strptime(date_string, "%Y-%m-%d")
|
||
return True
|
||
except ValueError:
|
||
# If parsing fails, it's not a valid date in the format YYYY-MM-DD
|
||
return False
|
||
|
||
def is_valid_date_range(self, date_range: str) -> bool:
|
||
try:
|
||
# Attempt to parse the string as a date range in the format
|
||
# YYYY-MM-DD:YYYY-MM-DD
|
||
start_date, end_date = date_range.split(":")
|
||
datetime.strptime(start_date.strip(), "%Y-%m-%d")
|
||
datetime.strptime(end_date.strip(), "%Y-%m-%d")
|
||
return True
|
||
except ValueError:
|
||
# If parsing fails, it's not a valid date range in the format
|
||
# YYYY-MM-DD:YYYY-MM-DD
|
||
return False
|
||
|
||
def get_offset_custom_or_inherit(
|
||
self,
|
||
offset: str,
|
||
outer_from_dttm: datetime,
|
||
outer_to_dttm: datetime,
|
||
) -> str:
|
||
"""
|
||
Get the time offset for custom or inherit.
|
||
|
||
:param offset: The offset string.
|
||
:param outer_from_dttm: The outer from datetime.
|
||
:param outer_to_dttm: The outer to datetime.
|
||
:returns: The time offset.
|
||
"""
|
||
if offset == "inherit":
|
||
# Shift back by the full length of the range, so the comparison
|
||
# covers the period immediately preceding it. The duration is
|
||
# expressed in seconds rather than days because ``timedelta.days``
|
||
# truncates: a 12-hour range would yield "0 days ago" and compare
|
||
# the range against itself, and a 36-hour range would shift by a
|
||
# single day and overlap it. Whole-day ranges resolve to the same
|
||
# instant either way.
|
||
duration = outer_to_dttm - outer_from_dttm
|
||
return f"{int(duration.total_seconds())} seconds ago"
|
||
if self.is_valid_date(offset):
|
||
# return the offset as the difference in days between the outer from dttm and the offset date (which is a YYYY-MM-DD string) formatted as a string with the " days ago" suffix # noqa: E501
|
||
offset_date = datetime.strptime(offset, "%Y-%m-%d")
|
||
return f"{(outer_from_dttm - offset_date).days} days ago"
|
||
return ""
|
||
|
||
def _get_temporal_column_for_filter( # noqa: C901
|
||
self, query_object: QueryObject, x_axis_label: str | None
|
||
) -> str | None:
|
||
"""
|
||
Helper method to reliably determine the temporal column for filtering.
|
||
|
||
This method tries multiple strategies to find the correct temporal column:
|
||
1. Use the column from existing TEMPORAL_RANGE filter
|
||
2. Use explicitly set granularity
|
||
3. Use x_axis_label if it exists
|
||
|
||
:param query_object: The query object
|
||
:param x_axis_label: The x-axis label from the query
|
||
:return: The name of the temporal column, or None if not found
|
||
"""
|
||
# Strategy 1: Use the column from existing TEMPORAL_RANGE filter
|
||
if query_object.filter:
|
||
for flt in query_object.filter:
|
||
if flt.get("op") == FilterOperator.TEMPORAL_RANGE:
|
||
col = flt.get("col")
|
||
if isinstance(col, str):
|
||
return col
|
||
elif isinstance(col, dict) and col.get("sqlExpression"):
|
||
return str(col.get("label") or col.get("sqlExpression"))
|
||
|
||
# Strategy 2: Use explicitly set granularity
|
||
if query_object.granularity:
|
||
return query_object.granularity
|
||
|
||
# Strategy 3: Use x_axis_label if it exists
|
||
if x_axis_label:
|
||
return x_axis_label
|
||
|
||
return None
|
||
|
||
def _process_date_range_offset(
|
||
self, offset_df: pd.DataFrame, join_keys: list[str]
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
"""Process date range offset data and return modified DataFrame and keys."""
|
||
temporal_cols = ["ds", "__timestamp", "dttm"]
|
||
non_temporal_join_keys = [key for key in join_keys if key not in temporal_cols]
|
||
|
||
if non_temporal_join_keys:
|
||
return offset_df, non_temporal_join_keys
|
||
|
||
metric_columns = [col for col in offset_df.columns if col not in temporal_cols]
|
||
|
||
if metric_columns:
|
||
aggregated_values = {}
|
||
for col in metric_columns:
|
||
if pd.api.types.is_numeric_dtype(offset_df[col]):
|
||
aggregated_values[col] = offset_df[col].sum()
|
||
else:
|
||
aggregated_values[col] = (
|
||
offset_df[col].iloc[0] if not offset_df.empty else None
|
||
)
|
||
|
||
offset_df = pd.DataFrame([aggregated_values])
|
||
|
||
return offset_df, []
|
||
|
||
def _apply_cleanup_logic(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset: str,
|
||
time_grain: str | None,
|
||
join_keys: list[str],
|
||
is_date_range_offset: bool,
|
||
) -> pd.DataFrame:
|
||
"""Apply appropriate cleanup logic based on offset type."""
|
||
if time_grain and not is_date_range_offset:
|
||
if join_keys:
|
||
col = df.pop(join_keys[0])
|
||
df.insert(0, col.name, col)
|
||
|
||
df.drop(
|
||
list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")),
|
||
axis=1,
|
||
inplace=True,
|
||
)
|
||
elif is_date_range_offset:
|
||
df.drop(
|
||
list(df.filter(regex=f"{R_SUFFIX}")),
|
||
axis=1,
|
||
inplace=True,
|
||
)
|
||
else:
|
||
df.drop(
|
||
list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")),
|
||
axis=1,
|
||
inplace=True,
|
||
)
|
||
|
||
return df
|
||
|
||
def _determine_join_keys(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset_df: pd.DataFrame,
|
||
offset: str,
|
||
time_grain: str | None,
|
||
join_keys: list[str],
|
||
is_date_range_offset: bool,
|
||
join_column_producer: Any,
|
||
x_axis_label: str | None = None,
|
||
x_axis_is_temporal: bool = False,
|
||
x_axis_datetime_format: str | None = None,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
"""Determine appropriate join keys and modify DataFrames if needed."""
|
||
if time_grain and not is_date_range_offset:
|
||
# When there are no join keys (e.g., totals query with columns=[]),
|
||
# check if the first column is a datetime before attempting to use it
|
||
# for join column generation. If it's not a datetime (e.g., only metric
|
||
# columns exist), fall back to empty join keys which will trigger the
|
||
# __temp_join_key__ mechanism in _perform_join.
|
||
if not join_keys and len(df.columns) > 0:
|
||
first_col_dtype = df.iloc[:, 0].dtype
|
||
if not pd.api.types.is_datetime64_any_dtype(first_col_dtype):
|
||
return offset_df, []
|
||
|
||
column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset
|
||
|
||
# Add offset join columns for relative time offsets
|
||
self.add_offset_join_column(
|
||
df, column_name, time_grain, offset, join_column_producer
|
||
)
|
||
self.add_offset_join_column(
|
||
offset_df, column_name, time_grain, None, join_column_producer
|
||
)
|
||
return offset_df, [column_name, *join_keys[1:]]
|
||
|
||
elif is_date_range_offset:
|
||
return self._process_date_range_offset(offset_df, join_keys)
|
||
|
||
else:
|
||
return self._align_offset_without_time_grain(
|
||
df,
|
||
offset_df,
|
||
offset,
|
||
join_keys,
|
||
x_axis_label,
|
||
x_axis_is_temporal,
|
||
x_axis_datetime_format,
|
||
)
|
||
|
||
def _align_offset_without_time_grain(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset_df: pd.DataFrame,
|
||
offset: str,
|
||
join_keys: list[str],
|
||
x_axis_label: str | None = None,
|
||
x_axis_is_temporal: bool = False,
|
||
x_axis_datetime_format: str | None = None,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
"""
|
||
Determine join keys for a relative offset when no time grain is set.
|
||
|
||
Without a time grain there is no truncated join column, but the two
|
||
series can still be aligned exactly: shifting the main series'
|
||
timestamps by the offset delta lands them on the offset series' raw
|
||
timestamps (normalize_time_delta returns a negative delta for "... ago"
|
||
offsets). Timestamps without an exact counterpart in the offset series
|
||
produce nulls, mirroring the grain-based join, and null timestamps in
|
||
the two series join to each other (both stringify to "NaT"). When
|
||
there is no temporal join key the original join keys are used as-is.
|
||
|
||
The shift is computed on wall-clock time, with any timezone dropped for
|
||
the duration of the alignment. The offset query's own time range was
|
||
shifted the same way -- ``get_past_or_future`` reads naive timestamps
|
||
-- so the rows it returns carry the source wall clock, and matching on
|
||
it is what aligns the two series. Re-localizing the result would only
|
||
reintroduce the DST edge cases that wall-clock arithmetic sidesteps:
|
||
shifting onto a skipped or repeated local hour raises out of pandas. A
|
||
single reading of a repeated hour aligns by wall clock. Distinct raw
|
||
values that normalize to the same working key are rejected because
|
||
they would fan out the merge; the error identifies a daylight-saving
|
||
fold when their UTC offsets differ.
|
||
|
||
A string-backed x-axis is parsed only when its physical dataset column
|
||
declares temporal metadata. Its configured datetime format is applied
|
||
to working join values without changing the displayed columns.
|
||
|
||
Month, quarter, and year offsets shift via ``DateOffset``, which clamps
|
||
to a valid calendar day (e.g. Mar 29, 30, and 31 all shift back one
|
||
month to Feb 28), so on daily/irregular data several end-of-month rows
|
||
can align to the same offset timestamp. This mirrors the inherent
|
||
ambiguity of "the same day N months ago" without a time grain to
|
||
truncate against.
|
||
"""
|
||
# Prefer the query's temporal x-axis when it is a join key; otherwise
|
||
# use the first datetime join key.
|
||
candidate_keys = sorted(join_keys, key=lambda key: key != x_axis_label)
|
||
temporal_join_key = next(
|
||
(
|
||
key
|
||
for key in candidate_keys
|
||
if key in df.columns
|
||
and key in offset_df.columns
|
||
and (
|
||
pd.api.types.is_datetime64_any_dtype(df[key])
|
||
or (key == x_axis_label and x_axis_is_temporal)
|
||
)
|
||
),
|
||
None,
|
||
)
|
||
if not temporal_join_key:
|
||
return offset_df, join_keys
|
||
|
||
source_values = df[temporal_join_key]
|
||
offset_values = offset_df[temporal_join_key]
|
||
raw_offset_values = offset_values.copy()
|
||
remaining_keys = [key for key in join_keys if key != temporal_join_key]
|
||
raw_duplicate_mask = offset_df.duplicated(
|
||
subset=[temporal_join_key, *remaining_keys], keep=False
|
||
)
|
||
if x_axis_is_temporal and (
|
||
not pd.api.types.is_datetime64_any_dtype(source_values)
|
||
or not pd.api.types.is_datetime64_any_dtype(offset_values)
|
||
):
|
||
if x_axis_datetime_format is None and any(
|
||
pd.api.types.is_numeric_dtype(values) and values.notna().any()
|
||
for values in (source_values, offset_values)
|
||
):
|
||
# A numeric temporal axis without a declared format cannot be
|
||
# interpreted, so retain the pre-alignment raw-key join.
|
||
return offset_df, join_keys
|
||
source_values = _coerce_temporal_join_series(
|
||
source_values,
|
||
temporal_join_key,
|
||
x_axis_datetime_format,
|
||
)
|
||
offset_values = _coerce_temporal_join_series(
|
||
offset_values,
|
||
temporal_join_key,
|
||
x_axis_datetime_format,
|
||
)
|
||
|
||
source = _as_wall_clock(source_values)
|
||
|
||
try:
|
||
delta: DateOffset | None = DateOffset(**normalize_time_delta(offset))
|
||
except (ValueError, TimeDeltaAmbiguousError):
|
||
delta = None
|
||
|
||
column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset
|
||
shifted = _shift_grainless_temporal_source(source, offset, delta)
|
||
|
||
# Join on string values so that mismatched key dtypes (e.g. an empty
|
||
# offset series materializes its join keys as NaN floats) cannot break
|
||
# the merge.
|
||
df[column_name] = shifted.map(str)
|
||
offset_df[column_name] = _as_wall_clock(offset_values).map(str)
|
||
|
||
actual_join_keys = [column_name, *remaining_keys]
|
||
normalized_duplicate_mask = offset_df.duplicated(
|
||
subset=actual_join_keys, keep=False
|
||
)
|
||
introduced_duplicate_mask = normalized_duplicate_mask & ~raw_duplicate_mask
|
||
if introduced_duplicate_mask.any():
|
||
if _has_multiple_utc_offsets(raw_offset_values[normalized_duplicate_mask]):
|
||
message = _(
|
||
"Unable to align time comparison because the offset series "
|
||
"contains an ambiguous daylight-saving fold for the same "
|
||
"dimensions and local time. Add a Time Grain or filter the "
|
||
"source to one UTC offset."
|
||
)
|
||
else:
|
||
message = _(
|
||
"Unable to align time comparison because the temporal axis "
|
||
"contains distinct values that normalize to the same instant "
|
||
"for the same dimensions. Standardize the source values or "
|
||
"add a Time Grain."
|
||
)
|
||
raise QueryObjectValidationError(message)
|
||
return offset_df, actual_join_keys
|
||
|
||
def _perform_join(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset_df: pd.DataFrame,
|
||
actual_join_keys: list[str],
|
||
how: Literal["left", "right", "inner", "outer"] = "left",
|
||
) -> pd.DataFrame:
|
||
"""Perform the appropriate join operation."""
|
||
if actual_join_keys:
|
||
return dataframe_utils.left_join_df(
|
||
left_df=df,
|
||
right_df=offset_df,
|
||
join_keys=actual_join_keys,
|
||
rsuffix=R_SUFFIX,
|
||
how=how,
|
||
)
|
||
else:
|
||
temp_key = "__temp_join_key__"
|
||
df[temp_key] = 1
|
||
offset_df[temp_key] = 1
|
||
|
||
result_df = dataframe_utils.left_join_df(
|
||
left_df=df,
|
||
right_df=offset_df,
|
||
join_keys=[temp_key],
|
||
rsuffix=R_SUFFIX,
|
||
how=how,
|
||
)
|
||
|
||
# Remove temporary join keys
|
||
result_df.drop(columns=[temp_key], inplace=True, errors="ignore")
|
||
result_df.drop(
|
||
columns=[f"{temp_key}{R_SUFFIX}"], inplace=True, errors="ignore"
|
||
)
|
||
return result_df
|
||
|
||
def join_offset_dfs(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset_dfs: dict[str, pd.DataFrame],
|
||
time_grain: str | None,
|
||
join_keys: list[str],
|
||
full_range: bool = False,
|
||
x_axis_label: str | None = None,
|
||
x_axis_is_temporal: bool = False,
|
||
x_axis_datetime_format: str | None = None,
|
||
) -> pd.DataFrame:
|
||
"""
|
||
Join offset DataFrames with the main DataFrame.
|
||
|
||
:param df: The main DataFrame.
|
||
:param offset_dfs: A list of offset DataFrames.
|
||
:param time_grain: The time grain used to calculate the temporal join key.
|
||
:param join_keys: The keys to join on.
|
||
:param full_range: When True, time-shifted (offset) series keep their full
|
||
time range instead of being truncated to the main series' range. This
|
||
uses an outer join so offset-only rows (e.g. the rest of a prior day when
|
||
the current day is still in progress) are preserved.
|
||
:param x_axis_label: The query's temporal x-axis label, used to pick the
|
||
temporal join key when no time grain is set.
|
||
:param x_axis_is_temporal: Whether physical-column metadata declares the
|
||
x-axis temporal.
|
||
:param x_axis_datetime_format: The physical x-axis column's configured
|
||
Python datetime format.
|
||
"""
|
||
join_column_producer = app.config["TIME_GRAIN_JOIN_COLUMN_PRODUCERS"].get(
|
||
time_grain
|
||
)
|
||
original_columns = list(df.columns)
|
||
|
||
for offset, offset_df in offset_dfs.items():
|
||
is_date_range_offset = self.is_valid_date_range(
|
||
offset
|
||
) and feature_flag_manager.is_feature_enabled(
|
||
"DATE_RANGE_TIMESHIFTS_ENABLED"
|
||
)
|
||
|
||
offset_df, actual_join_keys = self._determine_join_keys(
|
||
df,
|
||
offset_df,
|
||
offset,
|
||
time_grain,
|
||
join_keys,
|
||
is_date_range_offset,
|
||
join_column_producer,
|
||
x_axis_label,
|
||
x_axis_is_temporal,
|
||
x_axis_datetime_format,
|
||
)
|
||
|
||
# The full-range option is only meaningful for relative offsets aligned
|
||
# on a temporal join column (time_grain set). Date-range offsets and the
|
||
# grain-less path keep the existing left-join behavior.
|
||
use_outer_join = (
|
||
full_range
|
||
and bool(time_grain)
|
||
and not is_date_range_offset
|
||
and bool(join_keys)
|
||
)
|
||
how: Literal["left", "outer"] = "outer" if use_outer_join else "left"
|
||
|
||
df = self._perform_join(df, offset_df, actual_join_keys, how=how)
|
||
|
||
if use_outer_join:
|
||
df = self._coalesce_offset_index(df, offset, join_keys)
|
||
|
||
df = self._apply_cleanup_logic(
|
||
df, offset, time_grain, join_keys, is_date_range_offset
|
||
)
|
||
|
||
if not time_grain and not is_date_range_offset:
|
||
# The grain-less join indexes on the synthetic key plus the
|
||
# non-temporal join keys, which reset_index moves to the front;
|
||
# restore the original column order.
|
||
df = df[
|
||
[col for col in original_columns if col in df.columns]
|
||
+ [col for col in df.columns if col not in original_columns]
|
||
]
|
||
|
||
return df
|
||
|
||
def _coalesce_offset_index(
|
||
self,
|
||
df: pd.DataFrame,
|
||
offset: str,
|
||
join_keys: list[str],
|
||
) -> pd.DataFrame:
|
||
"""
|
||
Rebuild the temporal x-axis after an outer join with an offset DataFrame.
|
||
|
||
Offset-only rows (those with no matching row in the main series) have a null
|
||
x-axis value because the join happens on the normalized offset join column,
|
||
not the raw temporal column. Their real timestamp lives in the suffixed
|
||
right-hand column, expressed in the offset's own time range (e.g. "yesterday
|
||
15:00"). Shifting it forward by the offset places it on the main series'
|
||
axis (e.g. "today 15:00") so the comparison line spans the full period.
|
||
"""
|
||
x_axis = join_keys[0]
|
||
offset_x_axis = f"{x_axis}{R_SUFFIX}"
|
||
if x_axis not in df.columns or offset_x_axis not in df.columns:
|
||
return df
|
||
|
||
# normalize_time_delta returns a negative delta for "... ago" offsets, so
|
||
# subtracting it shifts the historical timestamp forward onto the main axis.
|
||
try:
|
||
forward_shift = DateOffset(**normalize_time_delta(offset))
|
||
except (ValueError, TimeDeltaAmbiguousError):
|
||
return df
|
||
|
||
shifted = df[offset_x_axis] - forward_shift
|
||
df[x_axis] = df[x_axis].fillna(shifted)
|
||
return df
|
||
|
||
def add_offset_join_column(
|
||
self,
|
||
df: pd.DataFrame,
|
||
name: str,
|
||
time_grain: str,
|
||
time_offset: str | None = None,
|
||
join_column_producer: Any = None,
|
||
) -> None:
|
||
"""
|
||
Adds an offset join column to the provided DataFrame.
|
||
|
||
The function modifies the DataFrame in-place.
|
||
|
||
:param df: pandas DataFrame to which the offset join column will be added.
|
||
:param name: The name of the new column to be added.
|
||
:param time_grain: The time grain used to calculate the new column.
|
||
:param time_offset: The time offset used to calculate the new column.
|
||
:param join_column_producer: A function to generate the join column.
|
||
"""
|
||
if join_column_producer:
|
||
df[name] = df.apply(lambda row: join_column_producer(row, 0), axis=1)
|
||
else:
|
||
df[name] = df.apply(
|
||
lambda row: self.generate_join_column(row, 0, time_grain, time_offset),
|
||
axis=1,
|
||
)
|
||
|
||
@staticmethod
|
||
def generate_join_column(
|
||
row: pd.Series,
|
||
column_index: int,
|
||
time_grain: str,
|
||
time_offset: str | None = None,
|
||
) -> str:
|
||
value = row.iloc[column_index]
|
||
|
||
if hasattr(value, "strftime"):
|
||
if time_offset and not ExploreMixin.is_valid_date_range_static(time_offset):
|
||
value = value + DateOffset(**normalize_time_delta(time_offset))
|
||
|
||
if time_grain in (
|
||
TimeGrain.WEEK_STARTING_SUNDAY,
|
||
TimeGrain.WEEK_ENDING_SATURDAY,
|
||
):
|
||
return value.strftime("%Y-W%U")
|
||
|
||
if time_grain in (
|
||
TimeGrain.WEEK,
|
||
TimeGrain.WEEK_STARTING_MONDAY,
|
||
TimeGrain.WEEK_ENDING_SUNDAY,
|
||
):
|
||
return value.strftime("%Y-W%W")
|
||
|
||
if time_grain == TimeGrain.MONTH:
|
||
return value.strftime("%Y-%m")
|
||
|
||
if time_grain == TimeGrain.QUARTER:
|
||
return value.strftime("%Y-Q") + str(value.quarter)
|
||
|
||
if time_grain == TimeGrain.YEAR:
|
||
return value.strftime("%Y")
|
||
|
||
return str(value)
|
||
|
||
@staticmethod
|
||
def is_valid_date_range_static(date_range: str) -> bool:
|
||
"""Static version of is_valid_date_range for use in static methods"""
|
||
try:
|
||
# Attempt to parse the string as a date range in the format
|
||
# YYYY-MM-DD:YYYY-MM-DD
|
||
start_date, end_date = date_range.split(":")
|
||
datetime.strptime(start_date.strip(), "%Y-%m-%d")
|
||
datetime.strptime(end_date.strip(), "%Y-%m-%d")
|
||
return True
|
||
except ValueError:
|
||
# If parsing fails, it's not a valid date range in the format
|
||
# YYYY-MM-DD:YYYY-MM-DD
|
||
return False
|
||
|
||
def get_rendered_sql(
|
||
self,
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
) -> str:
|
||
"""
|
||
Render sql with template engine (Jinja).
|
||
"""
|
||
if not self.sql:
|
||
return ""
|
||
|
||
sql = self.sql.strip("\t\r\n; ")
|
||
if template_processor:
|
||
try:
|
||
sql = template_processor.process_template(sql)
|
||
except UndefinedError as ex:
|
||
# Raised when a template references an undefined value, e.g.
|
||
# indexing into an empty list returned by `filter_values()`
|
||
# when no dashboard filter is active for that column.
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Virtual dataset template error: %(msg)s",
|
||
msg=str(ex),
|
||
)
|
||
) from ex
|
||
except (TemplateError, SupersetSyntaxErrorException) as ex:
|
||
# Extract error message from different exception types
|
||
if isinstance(ex, TemplateError):
|
||
error_msg = ex.message
|
||
else: # SupersetSyntaxErrorException
|
||
error_msg = str(ex.errors[0].message if ex.errors else ex)
|
||
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Error while rendering virtual dataset query: %(msg)s",
|
||
msg=error_msg,
|
||
)
|
||
) from ex
|
||
|
||
script = SQLScript(sql, engine=self.db_engine_spec.engine)
|
||
if len(script.statements) > 1:
|
||
raise QueryObjectValidationError(
|
||
_("Virtual dataset query cannot consist of multiple statements")
|
||
)
|
||
|
||
if not sql:
|
||
raise QueryObjectValidationError(_("Virtual dataset query cannot be empty"))
|
||
return sql
|
||
|
||
def text(self, clause: str) -> TextClause:
|
||
return self.db_engine_spec.get_text_clause(clause)
|
||
|
||
def get_from_clause(
|
||
self, template_processor: Optional[BaseTemplateProcessor] = None
|
||
) -> tuple[Union[TableClause, Alias], Optional[str]]:
|
||
"""
|
||
Return where to select the columns and metrics from. Either a physical table
|
||
or a virtual table with it's own subquery. If the FROM is referencing a
|
||
CTE, the CTE is returned as the second value in the return tuple.
|
||
|
||
For virtual datasets, RLS filters from underlying tables are applied to
|
||
prevent RLS bypass.
|
||
"""
|
||
from_sql = self.get_rendered_sql(template_processor) + "\n"
|
||
parsed_script = SQLScript(from_sql, engine=self.db_engine_spec.engine)
|
||
if parsed_script.has_mutation():
|
||
raise QueryObjectValidationError(
|
||
_("Virtual dataset query must be read-only")
|
||
)
|
||
|
||
# Apply RLS filters to virtual dataset SQL to prevent RLS bypass
|
||
# For each table referenced in the virtual dataset, apply its RLS filters
|
||
if parsed_script.statements:
|
||
default_schema = self.database.get_default_schema(self.catalog)
|
||
try:
|
||
rls_applied = False
|
||
# ``id`` lives on concrete subclasses (e.g. SqlaTable), not on
|
||
# ExploreMixin itself. getattr keeps this safe for non-dataset
|
||
# subclasses (e.g. SQL Lab Query), which have no RLS to dedupe.
|
||
self_id = getattr(self, "id", None)
|
||
for statement in parsed_script.statements:
|
||
if apply_rls(
|
||
self.database,
|
||
self.catalog,
|
||
self.schema or default_schema or "",
|
||
statement,
|
||
exclude_dataset_id=self_id,
|
||
):
|
||
rls_applied = True
|
||
|
||
# Only regenerate the SQL if RLS predicates were actually applied.
|
||
# Unnecessary round-tripping through sqlglot can alter SQL in
|
||
# dialect-specific ways (e.g. dropping column aliases, rewriting
|
||
# Redshift-specific syntax) and break virtual dataset queries.
|
||
if rls_applied:
|
||
from_sql = parsed_script.format()
|
||
|
||
except Exception as ex: # pylint: disable=broad-except
|
||
# RLS injection failures fail closed: only continue when it is
|
||
# positively confirmed that no RLS predicates apply to the
|
||
# referenced tables; any other outcome aborts the query.
|
||
try:
|
||
rls_required = any(
|
||
get_predicates_for_table(
|
||
table.qualify(
|
||
catalog=self.catalog,
|
||
schema=self.schema or default_schema or "",
|
||
),
|
||
self.database,
|
||
self.database.get_default_catalog(),
|
||
exclude_dataset_id=self_id,
|
||
)
|
||
for statement in parsed_script.statements
|
||
for table in statement.tables
|
||
)
|
||
except Exception: # pylint: disable=broad-except
|
||
rls_required = True
|
||
if rls_required:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Row-level security could not be applied to the "
|
||
"virtual dataset query, so it cannot be run "
|
||
"securely: %(msg)s",
|
||
msg=str(ex),
|
||
)
|
||
) from ex
|
||
logger.warning(
|
||
"RLS application to virtual dataset SQL failed, but no "
|
||
"predicates apply to its tables; continuing: %s",
|
||
ex,
|
||
)
|
||
|
||
from_sql = _normalize_mssql_virtual_dataset_sql(
|
||
from_sql, parsed_script, self.db_engine_spec.engine
|
||
)
|
||
|
||
cte = self.db_engine_spec.get_cte_query(from_sql)
|
||
from_clause = (
|
||
sa.table(self.db_engine_spec.cte_alias)
|
||
if cte
|
||
else TextAsFrom(self.text(from_sql), []).alias(VIRTUAL_TABLE_ALIAS)
|
||
)
|
||
|
||
return from_clause, cte
|
||
|
||
def adhoc_metric_to_sqla(
|
||
self,
|
||
metric: AdhocMetric,
|
||
columns_by_name: dict[str, "TableColumn"], # pylint: disable=unused-argument
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
processed: bool = False,
|
||
) -> ColumnElement:
|
||
"""
|
||
Turn an adhoc metric into a sqlalchemy column.
|
||
|
||
:param dict metric: Adhoc metric definition
|
||
:param dict columns_by_name: Columns for the current table
|
||
:param template_processor: template_processor instance
|
||
:param bool processed: Whether the sqlExpression has already been processed
|
||
:returns: The metric defined as a sqlalchemy column
|
||
:rtype: sqlalchemy.sql.column
|
||
"""
|
||
expression_type = metric.get("expressionType")
|
||
label = utils.get_metric_name(metric)
|
||
|
||
if expression_type == utils.AdhocMetricExpressionType.SIMPLE:
|
||
aggregate: Any = metric.get("aggregate")
|
||
metric_column = metric.get("column") or {}
|
||
column_name = cast(str, metric_column.get("column_name"))
|
||
sqla_column = sa.column(column_name)
|
||
|
||
if isinstance(aggregate, str) and aggregate in self.sqla_aggregations:
|
||
sqla_metric = self.sqla_aggregations[aggregate](sqla_column)
|
||
elif isinstance(aggregate, str) and (
|
||
extended_func := self.db_engine_spec.get_extended_aggregation_func(
|
||
aggregate
|
||
)
|
||
):
|
||
sqla_metric = extended_func(sqla_column)
|
||
elif (
|
||
isinstance(aggregate, str)
|
||
and aggregate in utils.EXTENDED_METRIC_AGGREGATES
|
||
):
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"The %(aggregate)s aggregate is not supported on this database",
|
||
aggregate=aggregate,
|
||
)
|
||
)
|
||
else:
|
||
raise QueryObjectValidationError(_("Adhoc metric aggregate is invalid"))
|
||
elif expression_type == utils.AdhocMetricExpressionType.SQL:
|
||
expression: Any = metric.get("sqlExpression")
|
||
if not isinstance(expression, str) or not expression.strip():
|
||
raise QueryObjectValidationError(
|
||
_("Adhoc metric SQL expression is invalid")
|
||
)
|
||
|
||
if not processed:
|
||
expression = self._process_select_expression(
|
||
expression=expression,
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
|
||
sqla_metric = literal_column(expression)
|
||
else:
|
||
raise QueryObjectValidationError("Adhoc metric expressionType is invalid")
|
||
|
||
return self.make_sqla_column_compatible(sqla_metric, label)
|
||
|
||
@property
|
||
def template_params_dict(self) -> dict[Any, Any]:
|
||
return {}
|
||
|
||
@staticmethod
|
||
def filter_values_handler( # pylint: disable=too-many-arguments # noqa: C901
|
||
values: Optional[FilterValues],
|
||
operator: str,
|
||
target_generic_type: utils.GenericDataType,
|
||
target_native_type: Optional[str] = None,
|
||
is_list_target: bool = False,
|
||
db_engine_spec: Optional[
|
||
builtins.type["BaseEngineSpec"]
|
||
] = None, # fix(hughhh): Optional[Type[BaseEngineSpec]]
|
||
db_extra: Optional[dict[str, Any]] = None,
|
||
) -> Optional[FilterValues]:
|
||
if values is None:
|
||
return None
|
||
|
||
temporal_comparison_operators: set[utils.FilterOperator] = {
|
||
utils.FilterOperator.EQUALS,
|
||
utils.FilterOperator.NOT_EQUALS,
|
||
utils.FilterOperator.IN,
|
||
utils.FilterOperator.NOT_IN,
|
||
utils.FilterOperator.GREATER_THAN,
|
||
utils.FilterOperator.LESS_THAN,
|
||
utils.FilterOperator.GREATER_THAN_OR_EQUALS,
|
||
utils.FilterOperator.LESS_THAN_OR_EQUALS,
|
||
}
|
||
|
||
def handle_temporal_value(value: FilterValue) -> FilterValue | ColumnElement:
|
||
if (
|
||
operator not in temporal_comparison_operators
|
||
or target_generic_type != utils.GenericDataType.TEMPORAL
|
||
or target_native_type is None
|
||
or db_engine_spec is None
|
||
):
|
||
return value
|
||
|
||
if isinstance(value, (float, int)) and not isinstance(value, bool):
|
||
epoch_ms: float = value
|
||
elif isinstance(value, str) and re.fullmatch(r"[+-]?\d+", value):
|
||
epoch_ms = int(value)
|
||
else:
|
||
return value
|
||
|
||
try:
|
||
dttm = datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc).replace(
|
||
tzinfo=None
|
||
)
|
||
except (OverflowError, OSError, ValueError):
|
||
return value
|
||
|
||
temporal_sql = db_engine_spec.convert_dttm(
|
||
target_type=target_native_type,
|
||
dttm=dttm,
|
||
db_extra=db_extra,
|
||
)
|
||
return literal_column(temporal_sql) if temporal_sql is not None else value
|
||
|
||
def handle_single_value(value: Optional[FilterValue]) -> Optional[FilterValue]:
|
||
if operator == utils.FilterOperator.TEMPORAL_RANGE:
|
||
return value
|
||
if isinstance(value, str):
|
||
value = value.strip("\t\n")
|
||
|
||
if (
|
||
target_generic_type == utils.GenericDataType.NUMERIC
|
||
and operator
|
||
not in {
|
||
utils.FilterOperator.ILIKE,
|
||
utils.FilterOperator.LIKE,
|
||
utils.FilterOperator.NOT_ILIKE,
|
||
utils.FilterOperator.NOT_LIKE,
|
||
}
|
||
):
|
||
# For backwards compatibility and edge cases
|
||
# where a column data type might have changed
|
||
return utils.cast_to_num(value)
|
||
if value == NULL_STRING:
|
||
return None
|
||
if value == EMPTY_STRING:
|
||
return ""
|
||
value = handle_temporal_value(value)
|
||
elif value is not None:
|
||
value = handle_temporal_value(value)
|
||
if target_generic_type == utils.GenericDataType.BOOLEAN:
|
||
return utils.cast_to_boolean(value)
|
||
return value
|
||
|
||
if isinstance(values, (list, tuple)):
|
||
values = [handle_single_value(v) for v in values] # type: ignore
|
||
else:
|
||
values = handle_single_value(values)
|
||
if is_list_target and not isinstance(values, (tuple, list)):
|
||
values = [values] # type: ignore
|
||
elif not is_list_target and isinstance(values, (tuple, list)):
|
||
values = values[0] if values else None
|
||
return values
|
||
|
||
def get_query_str(self, query_obj: QueryObjectDict) -> str:
|
||
query_str_ext = self.get_query_str_extended(query_obj)
|
||
all_queries = query_str_ext.prequeries + [query_str_ext.sql]
|
||
return ";\n\n".join(all_queries) + ";"
|
||
|
||
def _get_series_orderby(
|
||
self,
|
||
series_limit_metric: Metric,
|
||
metrics_by_name: dict[str, "SqlMetric"],
|
||
columns_by_name: dict[str, "TableColumn"],
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
) -> Column:
|
||
if utils.is_adhoc_metric(series_limit_metric):
|
||
assert isinstance(series_limit_metric, dict)
|
||
ob = self.adhoc_metric_to_sqla(series_limit_metric, columns_by_name)
|
||
elif (
|
||
isinstance(series_limit_metric, str)
|
||
and series_limit_metric in metrics_by_name
|
||
):
|
||
ob = metrics_by_name[series_limit_metric].get_sqla_col(
|
||
template_processor=template_processor
|
||
)
|
||
else:
|
||
raise QueryObjectValidationError(
|
||
_("Metric '%(metric)s' does not exist", metric=series_limit_metric)
|
||
)
|
||
return ob
|
||
|
||
def _reapply_query_filters(
|
||
self,
|
||
qry: Select,
|
||
apply_fetch_values_predicate: bool,
|
||
template_processor: Optional[BaseTemplateProcessor],
|
||
granularity: str | None,
|
||
time_filters: list[ColumnElement],
|
||
where_clause_and: list[ColumnElement],
|
||
having_clause_and: list[ColumnElement],
|
||
) -> Select:
|
||
"""
|
||
Re-apply WHERE and HAVING clauses to a reconstructed query.
|
||
|
||
When group_others_when_limit_reached=True, the query is reconstructed
|
||
with sa.select(), losing previously applied filters. This method
|
||
re-applies those filters to maintain query correctness.
|
||
|
||
The WHERE clause includes: user filters, RLS filters, extra WHERE
|
||
clauses, and time range filters accumulated in where_clause_and
|
||
and time_filters.
|
||
|
||
:param qry: The reconstructed SQLAlchemy Select object
|
||
:param apply_fetch_values_predicate: Whether to apply fetch values predicate
|
||
:param template_processor: Template processor for dynamic filters
|
||
:param granularity: Time granularity (if None, time_filters not applied)
|
||
:param time_filters: Time-based filter conditions
|
||
:param where_clause_and: Accumulated WHERE clause conditions
|
||
:param having_clause_and: Accumulated HAVING clause conditions
|
||
:return: The query with filters re-applied
|
||
"""
|
||
if apply_fetch_values_predicate and self.fetch_values_predicate:
|
||
qry = qry.where(
|
||
self.get_fetch_values_predicate(template_processor=template_processor)
|
||
)
|
||
|
||
if granularity:
|
||
if time_filters or where_clause_and:
|
||
qry = qry.where(and_(*(time_filters + where_clause_and)))
|
||
else:
|
||
all_filters = time_filters + where_clause_and
|
||
if all_filters:
|
||
qry = qry.where(and_(*all_filters))
|
||
|
||
if having_clause_and:
|
||
qry = qry.having(and_(*having_clause_and))
|
||
|
||
return qry
|
||
|
||
def adhoc_column_to_sqla(
|
||
self,
|
||
col: AdhocColumn,
|
||
force_type_check: bool = False,
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
apply_dataset_offset: bool = False,
|
||
sql_shifted_temporal_labels: set[str] | None = None,
|
||
) -> tuple[ColumnElement, Optional[GenericDataType]]:
|
||
raise NotImplementedError()
|
||
|
||
def _get_top_groups(
|
||
self,
|
||
df: pd.DataFrame,
|
||
dimensions: list[str],
|
||
groupby_exprs: dict[str, Any],
|
||
columns_by_name: dict[str, "TableColumn"],
|
||
) -> ColumnElement:
|
||
groups = []
|
||
for _unused, row in df.iterrows():
|
||
group = []
|
||
for dimension in dimensions:
|
||
value = self._normalize_prequery_result_type(
|
||
row,
|
||
dimension,
|
||
columns_by_name,
|
||
)
|
||
|
||
group.append(groupby_exprs[dimension] == value)
|
||
groups.append(and_(*group))
|
||
|
||
return or_(*groups)
|
||
|
||
def _apply_series_others_grouping(
|
||
self,
|
||
select_exprs: list[Any],
|
||
groupby_all_columns: dict[str, Any],
|
||
groupby_series_columns: dict[str, Any],
|
||
condition_factory: Callable[[str, Any], Any],
|
||
) -> tuple[list[Any], dict[str, Any]]:
|
||
"""
|
||
Apply "Others" grouping to series columns in both SELECT and GROUP BY clauses.
|
||
|
||
This method encapsulates the common logic for replacing series columns with
|
||
CASE expressions that group remaining series into an "Others" category when
|
||
the series limit is reached.
|
||
|
||
Args:
|
||
select_exprs: List of SELECT expressions to modify
|
||
groupby_all_columns: Dict of GROUP BY columns to modify
|
||
groupby_series_columns: Dict of series columns to apply Others grouping to
|
||
condition_factory: Function that takes (col_name, original_expr) and returns
|
||
the condition for when to keep original value vs use "Others"
|
||
|
||
Returns:
|
||
Tuple of (modified_select_exprs, modified_groupby_all_columns)
|
||
"""
|
||
# Modify SELECT expressions
|
||
modified_select_exprs = []
|
||
for expr in select_exprs:
|
||
if hasattr(expr, "name") and expr.name in groupby_series_columns:
|
||
# Create condition for this column using the factory function
|
||
condition = condition_factory(expr.name, expr)
|
||
|
||
# Create CASE expression: condition true -> original, else "Others"
|
||
case_expr = sa.case((condition, expr), else_=sa.literal("Others"))
|
||
case_expr = self.make_sqla_column_compatible(case_expr, expr.name)
|
||
modified_select_exprs.append(case_expr)
|
||
else:
|
||
modified_select_exprs.append(expr)
|
||
|
||
# Modify GROUP BY expressions
|
||
modified_groupby_all_columns = {}
|
||
for col_name, gby_expr in groupby_all_columns.items():
|
||
if col_name in groupby_series_columns:
|
||
# Create condition for this column using the factory function
|
||
condition = condition_factory(col_name, gby_expr)
|
||
|
||
# Create CASE expression for groupby
|
||
case_expr = sa.case(
|
||
(condition, gby_expr),
|
||
else_=sa.literal("Others"),
|
||
)
|
||
# Don't apply make_sqla_column_compatible to GROUP BY expressions.
|
||
# When make_sqla_column_compatible adds a label to the expression,
|
||
# it can cause SQLAlchemy to incorrectly render string literals
|
||
# without quotes in the GROUP BY clause (e.g., "ELSE Others"
|
||
# instead of "ELSE 'Others'")
|
||
modified_groupby_all_columns[col_name] = case_expr
|
||
else:
|
||
modified_groupby_all_columns[col_name] = gby_expr
|
||
|
||
return modified_select_exprs, modified_groupby_all_columns
|
||
|
||
def dttm_sql_literal(self, dttm: datetime, col: "TableColumn") -> str:
|
||
"""Convert datetime object to a SQL expression string"""
|
||
|
||
sql = (
|
||
self.db_engine_spec.convert_dttm(col.type, dttm, db_extra=self.db_extra)
|
||
if col.type
|
||
else None
|
||
)
|
||
|
||
if sql:
|
||
return sql
|
||
|
||
tf = col.python_date_format
|
||
|
||
# Fallback to the default format (if defined).
|
||
if not tf and self.db_extra:
|
||
tf = self.db_extra.get("python_date_format_by_column_name", {}).get(
|
||
col.column_name
|
||
)
|
||
|
||
if tf:
|
||
if tf in {"epoch_ms", "epoch_s"}:
|
||
# In general, Superset works with timezone-naive datetime objects
|
||
# internally. However, timestamp() applies local timezone to
|
||
# timezone-naive datetime objects. Therefore, we have to be explicit
|
||
# about UTC before calling timestamp().
|
||
dttm_tz_aware = dttm
|
||
if dttm_tz_aware.tzinfo is None:
|
||
dttm_tz_aware = dttm_tz_aware.replace(tzinfo=timezone.utc)
|
||
|
||
seconds_since_epoch = int(dttm_tz_aware.timestamp())
|
||
if tf == "epoch_s":
|
||
return str(seconds_since_epoch)
|
||
return str(seconds_since_epoch * 1000)
|
||
return f"'{dttm.strftime(tf)}'"
|
||
|
||
return f"""'{dttm.strftime("%Y-%m-%d %H:%M:%S.%f")}'"""
|
||
|
||
def get_time_filter( # pylint: disable=too-many-arguments # noqa: C901
|
||
self,
|
||
time_col: "TableColumn",
|
||
start_dttm: Optional[sa.DateTime],
|
||
end_dttm: Optional[sa.DateTime],
|
||
time_grain: Optional[str] = None,
|
||
label: Optional[str] = "__time",
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
) -> Optional[ColumnElement]:
|
||
col = (
|
||
time_col.get_timestamp_expression(
|
||
time_grain=time_grain,
|
||
label=label,
|
||
template_processor=template_processor,
|
||
)
|
||
if time_grain
|
||
else self.convert_tbl_column_to_sqla_col(
|
||
time_col, label=label, template_processor=template_processor
|
||
)
|
||
)
|
||
|
||
# Resolve dataset-level time-boundary adjustments. A configured
|
||
# `extra.timezone` (an IANA name, DST-aware) takes precedence: naive UI
|
||
# boundaries are interpreted in that zone and converted to UTC for
|
||
# comparison with UTC-stored data. When no timezone is configured, fall
|
||
# back to the legacy "Hour Offset" field instead: displayed values are
|
||
# shifted by +offset hours (see normalize_df / DateColumn in
|
||
# superset.utils.core), but the time filter compares raw stored values, so
|
||
# bounds are shifted by -offset to stay consistent with what's displayed
|
||
# (#104810).
|
||
dataset_timezone = self.get_dataset_timezone()
|
||
|
||
if dataset_timezone and (start_dttm or end_dttm):
|
||
try:
|
||
tz = ZoneInfo(dataset_timezone)
|
||
|
||
# The datetimes from the UI are naive (no timezone info). We
|
||
# interpret them as being in the dataset's configured timezone and
|
||
# convert them to UTC for comparison with UTC-stored data.
|
||
if start_dttm is not None:
|
||
start_dttm = (
|
||
start_dttm.replace(tzinfo=tz)
|
||
.astimezone(timezone.utc)
|
||
.replace(tzinfo=None)
|
||
)
|
||
if end_dttm is not None:
|
||
end_dttm = (
|
||
end_dttm.replace(tzinfo=tz)
|
||
.astimezone(timezone.utc)
|
||
.replace(tzinfo=None)
|
||
)
|
||
except (ZoneInfoNotFoundError, TypeError):
|
||
logger.warning(
|
||
"Invalid timezone %r in dataset extra; falling back to Hour "
|
||
"Offset if configured",
|
||
dataset_timezone,
|
||
)
|
||
dataset_timezone = None
|
||
|
||
if not dataset_timezone and (offset_hours := getattr(self, "offset", 0) or 0):
|
||
offset_hours = get_effective_hours_offset(
|
||
self.db_engine_spec,
|
||
time_col.type,
|
||
offset_hours,
|
||
db_extra=self.db_extra,
|
||
)
|
||
if start_dttm is not None:
|
||
start_dttm = start_dttm - timedelta(hours=offset_hours)
|
||
if end_dttm is not None:
|
||
end_dttm = end_dttm - timedelta(hours=offset_hours)
|
||
|
||
l = [] # noqa: E741
|
||
if start_dttm:
|
||
l.append(
|
||
col
|
||
>= self.db_engine_spec.get_text_clause(
|
||
self.dttm_sql_literal(start_dttm, time_col)
|
||
)
|
||
)
|
||
if end_dttm:
|
||
l.append(
|
||
col
|
||
< self.db_engine_spec.get_text_clause(
|
||
self.dttm_sql_literal(end_dttm, time_col)
|
||
)
|
||
)
|
||
if not l:
|
||
return None
|
||
return and_(True, *l)
|
||
|
||
def values_for_column( # pylint: disable=too-many-locals
|
||
self,
|
||
column_name: str,
|
||
limit: int = 10000,
|
||
denormalize_column: bool = False,
|
||
array_elements: bool = False,
|
||
) -> list[Any]:
|
||
# denormalize column name before querying for values
|
||
# unless disabled in the dataset configuration
|
||
db_dialect = self.database.get_dialect()
|
||
column_name_ = (
|
||
self.database.db_engine_spec.denormalize_name(db_dialect, column_name)
|
||
if denormalize_column
|
||
else column_name
|
||
)
|
||
cols = {col.column_name: col for col in self.columns}
|
||
target_col = cols[column_name_]
|
||
tp = self.get_template_processor()
|
||
tbl, cte = self.get_from_clause(tp)
|
||
|
||
db_engine_spec = self.database.db_engine_spec
|
||
value_expr = target_col.get_sqla_col(template_processor=tp)
|
||
# For element-level operators (Contains any / Contains all) on a
|
||
# multi-value (array) column, suggest the distinct **elements** rather
|
||
# than distinct whole arrays by expanding the array first (e.g. ClickHouse
|
||
# arrayJoin). Only when the engine supports arrays and the column is
|
||
# actually an array column; otherwise fall back to whole-value suggestions.
|
||
if array_elements and db_engine_spec.supports_multivalue_columns:
|
||
col_spec = db_engine_spec.get_column_spec(native_type=target_col.type)
|
||
if col_spec and col_spec.generic_type == GenericDataType.MULTI_VALUE:
|
||
value_expr = db_engine_spec.array_explode(value_expr)
|
||
|
||
qry = (
|
||
sa.select(
|
||
# The alias (label) here is important because some dialects will
|
||
# automatically add a random alias to the projection because of the
|
||
# call to DISTINCT; others will uppercase the column names. This
|
||
# gives us a deterministic column name in the dataframe.
|
||
value_expr.label("column_values")
|
||
)
|
||
.select_from(tbl)
|
||
.distinct()
|
||
)
|
||
if limit:
|
||
qry = qry.limit(limit)
|
||
|
||
if self.fetch_values_predicate:
|
||
qry = qry.where(self.get_fetch_values_predicate(template_processor=tp))
|
||
|
||
rls_filters = self.get_sqla_row_level_filters(template_processor=tp)
|
||
if rls_filters:
|
||
qry = qry.where(and_(*rls_filters))
|
||
|
||
with self.database.get_sqla_engine(
|
||
catalog=self.catalog, schema=self.schema
|
||
) as engine:
|
||
sql = str(qry.compile(engine, compile_kwargs={"literal_binds": True}))
|
||
sql = self._apply_cte(sql, cte)
|
||
|
||
# pylint: disable=protected-access
|
||
if engine.dialect.identifier_preparer._double_percents:
|
||
sql = sql.replace("%%", "%")
|
||
|
||
sql = self.database.mutate_sql_based_on_config(sql)
|
||
|
||
with engine.connect() as con:
|
||
|
||
def run_query(query_sql: str) -> pd.DataFrame:
|
||
return pd.read_sql_query(sql=self.text(query_sql), con=con)
|
||
|
||
if not self.sql:
|
||
# Physical-table dataset: the filter-values statement is
|
||
# generated by Superset, so a read-limit rejection (e.g.
|
||
# ClickHouse max_rows_to_read) is retried once with the
|
||
# engine's bounded-read override. Virtual datasets embed
|
||
# user-authored SQL and stay governed by operator read
|
||
# limits.
|
||
df = self.database.run_with_sampling_read_limit_retry(
|
||
sql, run_query
|
||
)
|
||
else:
|
||
df = run_query(sql)
|
||
# replace NaN with None to ensure it can be serialized to JSON
|
||
df = df.replace({np.nan: None})
|
||
return df["column_values"].to_list()
|
||
|
||
def validate_expression(
|
||
self,
|
||
expression: str,
|
||
expression_type: SqlExpressionType = SqlExpressionType.WHERE,
|
||
) -> ValidationResultDict:
|
||
"""
|
||
Validate a SQL expression against this datasource.
|
||
|
||
:param expression: SQL expression to validate
|
||
:param expression_type: Type of expression (column, metric, where, having)
|
||
:return: Dict with validation result and any errors
|
||
"""
|
||
|
||
from superset.sql_validators.base import SQLValidationAnnotation
|
||
|
||
try:
|
||
# Process template
|
||
tp = self.get_template_processor()
|
||
processed_expression = self._process_expression_template(expression, tp)
|
||
|
||
# Apply the same parsing policy used for stored adhoc column and
|
||
# metric expressions (single statement, no set operations, and no
|
||
# sub-queries unless ALLOW_ADHOC_SUBQUERY is enabled), so expression
|
||
# validation follows one policy across the query pipeline. Imported
|
||
# locally to avoid a circular import with the connectors package.
|
||
from superset.connectors.sqla.models import validate_stored_expression
|
||
|
||
validate_stored_expression(
|
||
self.database, self.catalog, self.schema or "", processed_expression
|
||
)
|
||
|
||
# Build validation query
|
||
tbl, cte = self.get_from_clause(tp)
|
||
validation_query = self._build_validation_query(
|
||
processed_expression, expression_type
|
||
)
|
||
|
||
# Execute validation
|
||
return self._execute_validation_query(
|
||
validation_query, tbl, cte or "", tp, processed_expression
|
||
)
|
||
except Exception as ex:
|
||
# Convert any exception to validation error format
|
||
error_msg = str(getattr(ex, "orig", ex))
|
||
return ValidationResultDict(
|
||
valid=False,
|
||
errors=[
|
||
SQLValidationAnnotation(
|
||
message=error_msg,
|
||
line_number=1,
|
||
start_column=0,
|
||
end_column=len(expression),
|
||
).to_dict()
|
||
],
|
||
)
|
||
|
||
def _process_expression_template(
|
||
self, expression: str, tp: Optional[BaseTemplateProcessor]
|
||
) -> str:
|
||
"""Process expression through template processor. Raises on error."""
|
||
if not tp:
|
||
return expression
|
||
|
||
if hasattr(tp, "process_template"):
|
||
return tp.process_template(expression)
|
||
return expression
|
||
|
||
def _build_validation_query(
|
||
self, expression: str, expression_type: SqlExpressionType
|
||
) -> Select:
|
||
"""Build validation query based on expression type. Raises on error."""
|
||
if expression_type == SqlExpressionType.COLUMN:
|
||
return sa.select(sa.literal_column(expression).label("test_col"))
|
||
elif expression_type == SqlExpressionType.METRIC:
|
||
return sa.select(sa.literal_column(expression).label("test_metric"))
|
||
elif expression_type == SqlExpressionType.WHERE:
|
||
return sa.select(sa.literal(1)).where(sa.text(expression))
|
||
elif expression_type == SqlExpressionType.HAVING:
|
||
dummy_col = sa.literal("A").label("dummy")
|
||
return (
|
||
sa.select(dummy_col)
|
||
.group_by(sa.text("dummy"))
|
||
.having(sa.text(expression))
|
||
)
|
||
else:
|
||
raise ValueError(f"Unsupported expression type: {expression_type}")
|
||
|
||
def _execute_validation_query(
|
||
self,
|
||
validation_query: Select,
|
||
tbl: TableClause | Alias,
|
||
cte: str,
|
||
tp: Optional[BaseTemplateProcessor],
|
||
expression: str,
|
||
) -> ValidationResultDict:
|
||
"""Execute validation query and return result."""
|
||
# Add FROM clause and prevent execution
|
||
validation_query = validation_query.select_from(tbl).where(sa.literal(False))
|
||
|
||
# Apply row-level security filters
|
||
rls_filters = self.get_sqla_row_level_filters(template_processor=tp)
|
||
if rls_filters:
|
||
validation_query = validation_query.where(and_(*rls_filters))
|
||
|
||
with self.database.get_sqla_engine() as engine:
|
||
sql = str(
|
||
validation_query.compile(engine, compile_kwargs={"literal_binds": True})
|
||
)
|
||
sql = self._apply_cte(sql, cte)
|
||
|
||
sql = self.database.mutate_sql_based_on_config(sql)
|
||
|
||
# Execute to validate without fetching data
|
||
with engine.connect() as con:
|
||
con.execute(self.text(sql))
|
||
|
||
return ValidationResultDict(valid=True, errors=[])
|
||
|
||
def _validate_stored_expression(self, expression: str) -> str:
|
||
return validate_stored_expression_at_query_time(
|
||
expression,
|
||
self.database,
|
||
self.catalog,
|
||
self.schema,
|
||
self.db_engine_spec.engine,
|
||
)
|
||
|
||
def get_timestamp_expression(
|
||
self,
|
||
column: dict[str, Any],
|
||
time_grain: Optional[str],
|
||
label: Optional[str] = None,
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
) -> Union[TimestampExpression, Label]:
|
||
"""
|
||
Return a SQLAlchemy Core element representation of self to be used in a query.
|
||
|
||
:param column: column object
|
||
:param time_grain: Optional time grain, e.g. P1Y
|
||
:param label: alias/label that column is expected to have
|
||
:param template_processor: template processor
|
||
:return: A TimeExpression object wrapped in a Label if supported by db
|
||
"""
|
||
label = label or utils.DTTM_ALIAS
|
||
column_spec = self.db_engine_spec.get_column_spec(column.get("type"))
|
||
type_ = column_spec.sqla_type if column_spec else sa.DateTime
|
||
col = sa.column(column.get("column_name"), type_=type_)
|
||
|
||
if template_processor:
|
||
expression = template_processor.process_template(column["column_name"])
|
||
expression = self._validate_stored_expression(expression)
|
||
col = sa.literal_column(expression, type_=type_)
|
||
|
||
time_expr = self.db_engine_spec.get_timestamp_expr(col, None, time_grain)
|
||
return self.make_sqla_column_compatible(time_expr, label)
|
||
|
||
def convert_tbl_column_to_sqla_col(
|
||
self,
|
||
tbl_column: "TableColumn",
|
||
label: Optional[str] = None,
|
||
template_processor: Optional[BaseTemplateProcessor] = None,
|
||
) -> Column:
|
||
label = label or tbl_column.column_name
|
||
db_engine_spec = self.db_engine_spec
|
||
column_spec = db_engine_spec.get_column_spec(
|
||
tbl_column.type, db_extra=self.db_extra
|
||
)
|
||
type_ = column_spec.sqla_type if column_spec else None
|
||
if expression := tbl_column.expression:
|
||
if template_processor:
|
||
expression = template_processor.process_template(expression)
|
||
if expression != tbl_column.expression:
|
||
# Re-check the rendered expression before embedding it.
|
||
expression = validate_rendered_expression(
|
||
expression, self.database, self.catalog, self.schema
|
||
)
|
||
expression = self._validate_stored_expression(expression)
|
||
col = literal_column(expression, type_=type_)
|
||
else:
|
||
col = sa.column(tbl_column.column_name, type_=type_)
|
||
col = self.make_sqla_column_compatible(col, label)
|
||
return col
|
||
|
||
def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements # noqa: C901
|
||
self,
|
||
apply_fetch_values_predicate: bool = False,
|
||
columns: Optional[list[Column]] = None,
|
||
extras: Optional[dict[str, Any]] = None,
|
||
filter: Optional[ # pylint: disable=redefined-builtin
|
||
list[utils.QueryObjectFilterClause]
|
||
] = None,
|
||
from_dttm: Optional[datetime] = None,
|
||
granularity: Optional[str] = None,
|
||
groupby: Optional[list[Column]] = None,
|
||
inner_from_dttm: Optional[datetime] = None,
|
||
inner_to_dttm: Optional[datetime] = None,
|
||
is_rowcount: bool = False,
|
||
is_timeseries: bool = True,
|
||
metrics: Optional[list[Metric]] = None,
|
||
orderby: Optional[list[OrderBy]] = None,
|
||
order_desc: bool = True,
|
||
to_dttm: Optional[datetime] = None,
|
||
series_columns: Optional[list[Column]] = None,
|
||
series_limit: Optional[int] = None,
|
||
series_limit_metric: Optional[Metric] = None,
|
||
group_others_when_limit_reached: bool = False,
|
||
grouping_sets: Optional[list[list[str]]] = None,
|
||
row_limit: Optional[int] = None,
|
||
row_offset: Optional[int] = None,
|
||
timeseries_limit: Optional[int] = None,
|
||
timeseries_limit_metric: Optional[Metric] = None,
|
||
time_shift: Optional[str] = None,
|
||
) -> SqlaQuery:
|
||
"""Querying any sqla table from this common interface"""
|
||
if granularity not in self.dttm_cols and granularity is not None:
|
||
granularity = self.main_dttm_col
|
||
|
||
extras = extras or {}
|
||
time_grain = extras.get("time_grain_sqla")
|
||
|
||
# DB-specifc quoting for identifiers
|
||
with self.database.get_sqla_engine() as engine:
|
||
quote = engine.dialect.identifier_preparer.quote
|
||
|
||
template_kwargs = {
|
||
"columns": columns,
|
||
"groupby": groupby,
|
||
"metrics": metrics,
|
||
"row_limit": row_limit,
|
||
"row_offset": row_offset,
|
||
"time_column": granularity,
|
||
"time_grain": time_grain,
|
||
"table_columns": [col.column_name for col in self.columns],
|
||
"filter": filter,
|
||
}
|
||
columns = columns or []
|
||
groupby = groupby or []
|
||
rejected_adhoc_filters_columns: list[Union[str, ColumnTyping]] = []
|
||
applied_adhoc_filters_columns: list[Union[str, ColumnTyping]] = []
|
||
db_engine_spec = self.db_engine_spec
|
||
series_column_labels = [
|
||
db_engine_spec.make_label_compatible(column)
|
||
for column in utils.get_column_names(
|
||
columns=series_columns or [],
|
||
)
|
||
]
|
||
# deprecated, to be removed in 2.0
|
||
if is_timeseries and timeseries_limit:
|
||
series_limit = timeseries_limit
|
||
series_limit_metric = series_limit_metric or timeseries_limit_metric
|
||
template_kwargs.update(self.template_params_dict)
|
||
extra_cache_keys: list[Any] = []
|
||
template_kwargs["extra_cache_keys"] = extra_cache_keys
|
||
removed_filters: list[str] = []
|
||
applied_template_filters: list[str] = []
|
||
template_kwargs["removed_filters"] = removed_filters
|
||
template_kwargs["applied_filters"] = applied_template_filters
|
||
template_processor = self.get_template_processor(**template_kwargs)
|
||
prequeries: list[str] = []
|
||
sql_shifted_temporal_labels: set[str] = set()
|
||
orderby = orderby or []
|
||
need_groupby = bool(metrics is not None or groupby)
|
||
metrics = metrics or []
|
||
|
||
# For backward compatibility
|
||
if granularity not in self.dttm_cols and granularity is not None:
|
||
granularity = self.main_dttm_col
|
||
|
||
columns_by_name: dict[str, "TableColumn"] = {
|
||
col.column_name: col for col in self.columns
|
||
}
|
||
|
||
metrics_by_name: dict[str, "SqlMetric"] = {
|
||
m.metric_name: m for m in self.metrics
|
||
}
|
||
|
||
if not granularity and is_timeseries:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Datetime column not provided as part table configuration "
|
||
"and is required by this type of chart"
|
||
)
|
||
)
|
||
if not metrics and not columns and not groupby:
|
||
raise QueryObjectValidationError(_("Empty query?"))
|
||
|
||
metrics_exprs: list[ColumnElement] = []
|
||
for metric in metrics:
|
||
if utils.is_adhoc_metric(metric):
|
||
assert isinstance(metric, dict)
|
||
metrics_exprs.append(
|
||
self.adhoc_metric_to_sqla(
|
||
metric=metric,
|
||
columns_by_name=columns_by_name,
|
||
template_processor=template_processor,
|
||
)
|
||
)
|
||
elif isinstance(metric, str) and metric in metrics_by_name:
|
||
metrics_exprs.append(
|
||
metrics_by_name[metric].get_sqla_col(
|
||
template_processor=template_processor
|
||
)
|
||
)
|
||
else:
|
||
raise QueryObjectValidationError(
|
||
_("Metric '%(metric)s' does not exist", metric=metric)
|
||
)
|
||
|
||
if metrics_exprs:
|
||
main_metric_expr = metrics_exprs[0]
|
||
else:
|
||
main_metric_expr, label = literal_column("COUNT(*)"), "ccount"
|
||
main_metric_expr = self.make_sqla_column_compatible(main_metric_expr, label)
|
||
|
||
# To ensure correct handling of the ORDER BY labeling we need to reference the
|
||
# metric instance if defined in the SELECT clause.
|
||
# use the key of the ColumnClause for the expected label
|
||
metrics_exprs_by_label = {m.key: m for m in metrics_exprs}
|
||
metrics_exprs_by_expr = {str(m): m for m in metrics_exprs}
|
||
adhoc_columns_by_label: dict[str, AdhocColumn] = {}
|
||
for selected in columns:
|
||
if not utils.is_adhoc_column(selected):
|
||
continue
|
||
selected_label = selected.get("label")
|
||
if isinstance(selected_label, str) and selected_label:
|
||
adhoc_columns_by_label[selected_label] = selected
|
||
|
||
# Since orderby may use adhoc metrics, too; we need to process them first
|
||
orderby_exprs: list[ColumnElement] = []
|
||
for orig_col, ascending in orderby: # noqa: B007
|
||
col: Union[AdhocMetric, ColumnElement] = orig_col
|
||
if isinstance(col, dict):
|
||
# process a copy, as the dict is shared with `QueryObject.orderby`
|
||
# and `QueryContext.cache_values`; writing the processed expression
|
||
# back would change the cache key of a rehydrated query context
|
||
col = cast(AdhocMetric, dict(col))
|
||
if col.get("sqlExpression"):
|
||
col["sqlExpression"] = self._process_orderby_expression(
|
||
expression=col["sqlExpression"],
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
if utils.is_adhoc_metric(col):
|
||
# add adhoc sort by column to columns_by_name if not exists
|
||
# Pass the template processor so a SIMPLE adhoc metric that
|
||
# references a calculated column has that column's Jinja
|
||
# expression rendered, matching SELECT/WHERE/GROUP BY. This
|
||
# is a no-op for the SQL expression type, whose
|
||
# `sqlExpression` was already rendered above; `processed`
|
||
# only guards that branch.
|
||
col = self.adhoc_metric_to_sqla(
|
||
col,
|
||
columns_by_name,
|
||
template_processor=template_processor,
|
||
processed=True,
|
||
)
|
||
# use the existing instance, if possible
|
||
col = metrics_exprs_by_expr.get(str(col), col)
|
||
need_groupby = True
|
||
elif col in metrics_exprs_by_label:
|
||
col = metrics_exprs_by_label[col]
|
||
need_groupby = True
|
||
elif isinstance(col, str) and col in adhoc_columns_by_label:
|
||
col, _unused = self.adhoc_column_to_sqla(
|
||
col=adhoc_columns_by_label[col],
|
||
template_processor=template_processor,
|
||
apply_dataset_offset=True,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|
||
elif col in metrics_by_name:
|
||
col = metrics_by_name[col].get_sqla_col(
|
||
template_processor=template_processor
|
||
)
|
||
need_groupby = True
|
||
elif col in columns_by_name:
|
||
col = self.convert_tbl_column_to_sqla_col(
|
||
columns_by_name[col], template_processor=template_processor
|
||
)
|
||
|
||
if isinstance(col, ColumnElement):
|
||
orderby_exprs.append(col)
|
||
else:
|
||
# Could not convert a column reference to valid ColumnElement
|
||
raise QueryObjectValidationError(
|
||
_("Unknown column used in orderby: %(col)s", col=orig_col)
|
||
)
|
||
|
||
select_exprs: list[Union[Column, Label]] = []
|
||
groupby_all_columns = {}
|
||
groupby_series_columns = {}
|
||
|
||
# filter out the pseudo column __timestamp from columns
|
||
columns = [col for col in columns if col != utils.DTTM_ALIAS]
|
||
dttm_col = columns_by_name.get(granularity) if granularity else None
|
||
|
||
if need_groupby:
|
||
# dedup columns while preserving order
|
||
columns = groupby or columns
|
||
for selected in columns:
|
||
if isinstance(selected, str):
|
||
# if groupby field/expr equals granularity field/expr
|
||
if selected == granularity:
|
||
table_col = columns_by_name[selected]
|
||
outer = table_col.get_timestamp_expression(
|
||
time_grain=time_grain,
|
||
label=selected,
|
||
template_processor=template_processor,
|
||
apply_dataset_offset=True,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|
||
# if groupby field equals a selected column
|
||
elif selected in columns_by_name:
|
||
outer = self.convert_tbl_column_to_sqla_col(
|
||
columns_by_name[selected],
|
||
template_processor=template_processor,
|
||
)
|
||
else:
|
||
selected = self._process_select_expression(
|
||
expression=selected,
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
outer = literal_column(f"({selected})")
|
||
outer = self.make_sqla_column_compatible(outer, selected)
|
||
else:
|
||
outer, _unused = self.adhoc_column_to_sqla(
|
||
col=selected,
|
||
template_processor=template_processor,
|
||
apply_dataset_offset=True,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|
||
groupby_all_columns[outer.name] = outer
|
||
if (
|
||
is_timeseries and not series_column_labels
|
||
) or outer.name in series_column_labels:
|
||
groupby_series_columns[outer.name] = outer
|
||
select_exprs.append(outer)
|
||
elif columns:
|
||
for selected in columns:
|
||
# Resolve a known column directly to its ``TableColumn`` so that
|
||
# multi-part identifiers (e.g. a BigQuery STRUCT field registered
|
||
# with a dotted ``column_name`` such as ``a.b.c``) are quoted per
|
||
# segment, matching the chart/groupby selection path above.
|
||
# Without this, a registered string column falls through to the
|
||
# ``quote()`` + ``_process_select_expression`` (sqlglot
|
||
# ``sanitize_clause``) path below, which re-serializes the merged
|
||
# identifier into a table-qualified form that no longer matches
|
||
# ``columns_by_name`` and is emitted via ``literal_column``
|
||
# as a single quoted name — breaking drill to detail / samples
|
||
# queries on nested columns (SC-111745).
|
||
# The guard fires for every registered physical column, not only
|
||
# dotted ones; that is intentional — for a plain name like ``id``
|
||
# the resolved column produces SQL identical to the fallback path.
|
||
if isinstance(selected, str) and selected in columns_by_name:
|
||
select_exprs.append(
|
||
self.convert_tbl_column_to_sqla_col(
|
||
columns_by_name[selected],
|
||
label=selected,
|
||
template_processor=template_processor,
|
||
)
|
||
)
|
||
continue
|
||
if is_adhoc_column(selected):
|
||
# Delegate to ``adhoc_column_to_sqla`` (same path as the
|
||
# ``need_groupby`` branch) so calculated columns are resolved
|
||
# via metadata lookup and their expression is inlined
|
||
# (#34784).
|
||
outer, _unused = self.adhoc_column_to_sqla(
|
||
col=selected,
|
||
template_processor=template_processor,
|
||
apply_dataset_offset=True,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|
||
select_exprs.append(outer)
|
||
continue
|
||
if isinstance(selected, str):
|
||
selected = self._process_select_expression(
|
||
expression=quote(selected),
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
select_exprs.append(
|
||
self.make_sqla_column_compatible(
|
||
literal_column(selected), selected
|
||
)
|
||
)
|
||
metrics_exprs = []
|
||
|
||
time_filters = []
|
||
|
||
# Process FROM clause early to populate removed_filters from virtual dataset
|
||
# templates before we decide whether to add time filters
|
||
tbl, cte = self.get_from_clause(template_processor)
|
||
|
||
if granularity:
|
||
if granularity not in columns_by_name or not dttm_col:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
'Time column "%(col)s" does not exist in dataset',
|
||
col=granularity,
|
||
)
|
||
)
|
||
|
||
if is_timeseries:
|
||
timestamp = dttm_col.get_timestamp_expression(
|
||
time_grain=time_grain,
|
||
template_processor=template_processor,
|
||
apply_dataset_offset=True,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|
||
# always put timestamp as the first column
|
||
select_exprs.insert(0, timestamp)
|
||
groupby_all_columns[timestamp.name] = timestamp
|
||
|
||
# Use main dttm column to support index with secondary dttm columns.
|
||
if (
|
||
self.always_filter_main_dttm
|
||
and self.main_dttm_col in self.dttm_cols
|
||
and self.main_dttm_col != dttm_col.column_name
|
||
and self.main_dttm_col not in removed_filters
|
||
):
|
||
_main_dttm_filter = self.get_time_filter(
|
||
time_col=columns_by_name[self.main_dttm_col],
|
||
start_dttm=from_dttm,
|
||
end_dttm=to_dttm,
|
||
template_processor=template_processor,
|
||
)
|
||
if _main_dttm_filter is not None:
|
||
time_filters.append(_main_dttm_filter)
|
||
|
||
# Check if time filter should be skipped because it was handled in template.
|
||
# Check both the actual column name and __timestamp alias
|
||
should_skip_time_filter = (
|
||
dttm_col.column_name in removed_filters
|
||
or utils.DTTM_ALIAS in removed_filters
|
||
)
|
||
|
||
if not should_skip_time_filter:
|
||
time_filter_column = self.get_time_filter(
|
||
time_col=dttm_col,
|
||
start_dttm=from_dttm,
|
||
end_dttm=to_dttm,
|
||
template_processor=template_processor,
|
||
)
|
||
if time_filter_column is not None:
|
||
time_filters.append(time_filter_column)
|
||
|
||
# Gate on `groupby_all_columns` rather than the raw dimensions: it is the
|
||
# real GROUP BY signal and also captures the timeseries time bucket. A
|
||
# non-aggregate Custom SQL metric under a GROUP BY is invalid SQL, so
|
||
# raise a clear error instead of a raw database one (#38913). Templated
|
||
# expressions may render to an aggregate at runtime, so leave them alone.
|
||
if groupby_all_columns:
|
||
for metric in metrics:
|
||
if (
|
||
utils.is_adhoc_metric(metric)
|
||
and metric.get("expressionType")
|
||
== utils.AdhocMetricExpressionType.SQL
|
||
):
|
||
expression = metric.get("sqlExpression")
|
||
if (
|
||
isinstance(expression, str)
|
||
and "{{" not in expression
|
||
and "{%" not in expression
|
||
and not has_aggregate(expression, self.database.backend)
|
||
):
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
'The Custom SQL metric "%(metric)s" is not an '
|
||
"aggregate and can't be combined with a GROUP BY. "
|
||
"Wrap it in an aggregate function, e.g. "
|
||
"%(example)s.",
|
||
metric=utils.get_metric_name(metric),
|
||
example="MAX(%s)" % expression,
|
||
)
|
||
)
|
||
|
||
# Always remove duplicates by column name, as sometimes `metrics_exprs`
|
||
# can have the same name as a groupby column (e.g. when users use
|
||
# raw columns as custom SQL adhoc metric).
|
||
select_exprs = remove_duplicates(
|
||
select_exprs + metrics_exprs, key=lambda x: x.name
|
||
)
|
||
|
||
# GROUPING SETS collapse (SIP.md, phase 3b): when the request supplies
|
||
# rollup levels via `grouping_sets` and the engine supports it, compute
|
||
# every level in one query and tag each row with GROUPING() markers so
|
||
# the caller can split the result back per level. Strictly opt-in: with
|
||
# `grouping_sets` unset the query is byte-identical to before.
|
||
use_grouping_sets = bool(
|
||
grouping_sets
|
||
and groupby_all_columns
|
||
and db_engine_spec.supports_grouping_sets
|
||
)
|
||
if use_grouping_sets:
|
||
select_exprs = select_exprs + [
|
||
grouping_id_column(gby_expr, grouping_marker_label(name))
|
||
for name, gby_expr in groupby_all_columns.items()
|
||
]
|
||
|
||
# Expected output columns
|
||
labels_expected = [c.key for c in select_exprs]
|
||
|
||
# Order by columns are "hidden" columns, some databases require them
|
||
# always be present in SELECT if an aggregation function is used
|
||
if not db_engine_spec.allows_hidden_orderby_agg:
|
||
select_exprs = remove_duplicates(select_exprs + orderby_exprs)
|
||
|
||
qry = sa.select(*select_exprs)
|
||
|
||
if groupby_all_columns:
|
||
if use_grouping_sets:
|
||
gs_levels = [
|
||
[
|
||
groupby_all_columns[col]
|
||
for col in level
|
||
if col in groupby_all_columns
|
||
]
|
||
for level in grouping_sets or []
|
||
]
|
||
qry = qry.group_by(grouping_sets_clause(gs_levels))
|
||
else:
|
||
qry = qry.group_by(*groupby_all_columns.values())
|
||
|
||
where_clause_and: list[ColumnElement] = []
|
||
having_clause_and: list[ColumnElement] = []
|
||
|
||
for flt in filter or []:
|
||
if not all(flt.get(s) for s in ["col", "op"]):
|
||
continue
|
||
flt_col = flt["col"]
|
||
val = flt.get("val")
|
||
flt_grain = flt.get("grain")
|
||
op = utils.FilterOperator(flt["op"].upper())
|
||
col_obj: Optional["TableColumn"] = None
|
||
sqla_col: Optional[Column] = None
|
||
adhoc_generic_type: Optional[GenericDataType] = None
|
||
is_metric_filter = (
|
||
False # Track if this is a filter on a metric (needs HAVING clause)
|
||
)
|
||
if flt_col == utils.DTTM_ALIAS and is_timeseries and dttm_col:
|
||
col_obj = dttm_col
|
||
elif is_adhoc_column(flt_col):
|
||
try:
|
||
sqla_col, adhoc_generic_type = self.adhoc_column_to_sqla(
|
||
cast("AdhocColumn", flt_col),
|
||
force_type_check=True,
|
||
template_processor=template_processor,
|
||
)
|
||
applied_adhoc_filters_columns.append(flt_col)
|
||
except ColumnNotFoundException:
|
||
rejected_adhoc_filters_columns.append(flt_col)
|
||
continue
|
||
else:
|
||
# Check if it's a regular column first
|
||
col_obj = columns_by_name.get(cast(str, flt_col))
|
||
# If not found in columns, check if it's a metric
|
||
# This supports filtering on metric columns for any chart type
|
||
if col_obj is None:
|
||
# Fall back to verbose_name so filters produced by
|
||
# right-click "Drill to detail by" (which can pass a
|
||
# column's verbose label) still resolve to a real column.
|
||
col_obj = next(
|
||
(c for c in self.columns if c.verbose_name == flt_col),
|
||
None,
|
||
)
|
||
if (
|
||
col_obj is None
|
||
and isinstance(flt_col, str)
|
||
and flt_col in metrics_by_name
|
||
):
|
||
# Convert metric to SQLA column expression
|
||
sqla_col = metrics_by_name[flt_col].get_sqla_col(
|
||
template_processor=template_processor
|
||
)
|
||
is_metric_filter = True
|
||
filter_grain = flt.get("grain")
|
||
|
||
# Check if this filter should be skipped because it was handled in
|
||
# template. Special handling for __timestamp alias: check both the
|
||
# alias and the actual column name
|
||
filter_col_name = get_column_name(flt_col)
|
||
should_skip_filter = filter_col_name in removed_filters
|
||
if not should_skip_filter and flt_col == utils.DTTM_ALIAS and col_obj:
|
||
# For __timestamp, also check if the actual datetime column was
|
||
# removed
|
||
should_skip_filter = col_obj.column_name in removed_filters
|
||
|
||
if should_skip_filter:
|
||
# Skip generating SQLA filter when the jinja template handles it.
|
||
continue
|
||
|
||
# Determine which clause list to use: HAVING for metrics, WHERE for columns
|
||
# Metric filters use HAVING clause because they involve aggregate functions
|
||
target_clause_list = (
|
||
having_clause_and if is_metric_filter else where_clause_and
|
||
)
|
||
|
||
if col_obj or sqla_col is not None:
|
||
if sqla_col is not None:
|
||
pass
|
||
elif col_obj and filter_grain:
|
||
sqla_col = col_obj.get_timestamp_expression(
|
||
time_grain=filter_grain, template_processor=template_processor
|
||
)
|
||
elif col_obj:
|
||
sqla_col = self.convert_tbl_column_to_sqla_col(
|
||
tbl_column=col_obj, template_processor=template_processor
|
||
)
|
||
# Parenthesize expression-based columns to prevent operator
|
||
# precedence issues (e.g. OR in a calculated column breaking
|
||
# surrounding AND filters). Same pattern as extras.where
|
||
# wrapping added in PR #38183.
|
||
if sqla_col is not None and (
|
||
(col_obj and col_obj.expression) or is_adhoc_column(flt_col)
|
||
):
|
||
sqla_col = Grouping(sqla_col)
|
||
col_type = col_obj.type if col_obj else None
|
||
col_spec = db_engine_spec.get_column_spec(native_type=col_type)
|
||
is_multivalue_col = bool(
|
||
col_spec and col_spec.generic_type == GenericDataType.MULTI_VALUE
|
||
)
|
||
# Element type of an array column (e.g. Array(Int32) -> NUMERIC),
|
||
# used to coerce filter values before building array expressions.
|
||
array_element_type = (
|
||
db_engine_spec.get_array_element_type(col_type)
|
||
if is_multivalue_col
|
||
else None
|
||
)
|
||
is_list_target = op in (
|
||
utils.FilterOperator.IN,
|
||
utils.FilterOperator.NOT_IN,
|
||
utils.FilterOperator.CONTAINS_ANY,
|
||
utils.FilterOperator.CONTAINS_ALL,
|
||
)
|
||
|
||
col_advanced_data_type = col_obj.advanced_data_type if col_obj else ""
|
||
|
||
if col_spec and not col_advanced_data_type:
|
||
target_generic_type = col_spec.generic_type
|
||
elif adhoc_generic_type is not None and not col_advanced_data_type:
|
||
# Adhoc columns have no TableColumn metadata; fall back to
|
||
# the generic type resolved by adhoc_column_to_sqla so
|
||
# filter values get coerced to match the SQL expression
|
||
# (e.g. numeric IN-lists stay unquoted when the expression
|
||
# casts to BIGINT).
|
||
target_generic_type = adhoc_generic_type
|
||
else:
|
||
target_generic_type = GenericDataType.STRING
|
||
eq = self.filter_values_handler(
|
||
values=val,
|
||
operator=op,
|
||
target_generic_type=target_generic_type,
|
||
target_native_type=col_type,
|
||
is_list_target=is_list_target,
|
||
db_engine_spec=db_engine_spec,
|
||
db_extra=self.db_extra,
|
||
)
|
||
|
||
# Get ADVANCED_DATA_TYPES from config when needed
|
||
ADVANCED_DATA_TYPES = app.config.get("ADVANCED_DATA_TYPES", {}) # noqa: N806
|
||
|
||
if (
|
||
col_advanced_data_type != ""
|
||
and feature_flag_manager.is_feature_enabled(
|
||
"ENABLE_ADVANCED_DATA_TYPES"
|
||
)
|
||
and col_advanced_data_type in ADVANCED_DATA_TYPES
|
||
):
|
||
values = eq if is_list_target else [eq] # type: ignore
|
||
bus_resp: AdvancedDataTypeResponse = ADVANCED_DATA_TYPES[
|
||
col_advanced_data_type
|
||
].translate_type(
|
||
{
|
||
"type": col_advanced_data_type,
|
||
"values": values,
|
||
}
|
||
)
|
||
if bus_resp["error_message"]:
|
||
raise AdvancedDataTypeResponseError(
|
||
_(bus_resp["error_message"])
|
||
)
|
||
|
||
target_clause_list.append(
|
||
ADVANCED_DATA_TYPES[col_advanced_data_type].translate_filter(
|
||
sqla_col, op, bus_resp["values"]
|
||
)
|
||
)
|
||
elif is_multivalue_col and op in {
|
||
utils.FilterOperator.EQUALS,
|
||
utils.FilterOperator.NOT_EQUALS,
|
||
utils.FilterOperator.IN,
|
||
utils.FilterOperator.NOT_IN,
|
||
}:
|
||
# Whole-array (column-level) comparison against array
|
||
# literal(s). The value is a pasted array literal like
|
||
# ``['a', 'b']`` (parsed into elements): ``col = ['a', 'b']``
|
||
# for = / !=; for IN / NOT IN each entered value is one such
|
||
# array literal (``col IN (['a'], ['b'])``).
|
||
if op in {
|
||
utils.FilterOperator.EQUALS,
|
||
utils.FilterOperator.NOT_EQUALS,
|
||
}:
|
||
literal = db_engine_spec.array_literal(
|
||
coerce_array_values(
|
||
parse_array_literal(val), array_element_type
|
||
)
|
||
)
|
||
cond = (
|
||
sqla_col != literal
|
||
if op == utils.FilterOperator.NOT_EQUALS
|
||
else sqla_col == literal
|
||
)
|
||
else:
|
||
candidates: list[Any] = (
|
||
list(val) if isinstance(val, (list, tuple)) else [val]
|
||
)
|
||
cond = sqla_col.in_(
|
||
[
|
||
db_engine_spec.array_literal(
|
||
coerce_array_values(
|
||
parse_array_literal(candidate),
|
||
array_element_type,
|
||
)
|
||
)
|
||
for candidate in candidates
|
||
]
|
||
)
|
||
if op == utils.FilterOperator.NOT_IN:
|
||
cond = ~cond
|
||
target_clause_list.append(cond)
|
||
elif op in {
|
||
utils.FilterOperator.IN,
|
||
utils.FilterOperator.NOT_IN,
|
||
}:
|
||
# CONTAINS_ANY/CONTAINS_ALL also produce a list ``eq`` (they
|
||
# are in ``is_list_target``), but are element-level array ops
|
||
# handled by their own branch below — not IN.
|
||
assert isinstance(eq, (tuple, list))
|
||
if len(eq) == 0:
|
||
raise QueryObjectValidationError(
|
||
_("Filter value list cannot be empty")
|
||
)
|
||
|
||
# Normalize mixed int/float values before binding, since
|
||
# SQLAlchemy may infer the bind parameter type from the
|
||
# first element and silently truncate other values
|
||
# (see #33206)
|
||
if target_generic_type == utils.GenericDataType.NUMERIC and any(
|
||
isinstance(v, float) for v in eq
|
||
):
|
||
eq = [
|
||
float(v) if isinstance(v, (int, float)) else v for v in eq
|
||
]
|
||
|
||
if len(eq) > len(
|
||
eq_without_none := [x for x in eq if x is not None]
|
||
):
|
||
is_null_cond = sqla_col.is_(None)
|
||
if eq:
|
||
cond = or_(is_null_cond, sqla_col.in_(eq_without_none))
|
||
else:
|
||
cond = is_null_cond
|
||
else:
|
||
cond = sqla_col.in_(eq)
|
||
|
||
if op == utils.FilterOperator.NOT_IN:
|
||
cond = ~cond
|
||
target_clause_list.append(cond)
|
||
elif op in {
|
||
utils.FilterOperator.IS_NULL,
|
||
utils.FilterOperator.IS_NOT_NULL,
|
||
}:
|
||
target_clause_list.append(
|
||
db_engine_spec.handle_null_filter(sqla_col, op)
|
||
)
|
||
elif op in {
|
||
utils.FilterOperator.IS_EMPTY,
|
||
utils.FilterOperator.IS_NOT_EMPTY,
|
||
}:
|
||
# Element-level array operators: length(col) == 0 / > 0.
|
||
if target_generic_type != GenericDataType.MULTI_VALUE:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"The %(op)s operator is only supported for "
|
||
"multi-value (array) columns.",
|
||
op=op,
|
||
)
|
||
)
|
||
length_expr = db_engine_spec.array_length(sqla_col)
|
||
if op == utils.FilterOperator.IS_EMPTY:
|
||
target_clause_list.append(length_expr == 0)
|
||
else:
|
||
target_clause_list.append(length_expr > 0)
|
||
elif op in {
|
||
utils.FilterOperator.LENGTH_EQUALS,
|
||
utils.FilterOperator.LENGTH_GREATER_THAN,
|
||
utils.FilterOperator.LENGTH_LESS_THAN,
|
||
utils.FilterOperator.LENGTH_GREATER_THAN_OR_EQUALS,
|
||
utils.FilterOperator.LENGTH_LESS_THAN_OR_EQUALS,
|
||
}:
|
||
# Length filter: compare the array's element count to a
|
||
# number, e.g. length(col) > 2.
|
||
if target_generic_type != GenericDataType.MULTI_VALUE:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"The %(op)s operator is only supported for "
|
||
"multi-value (array) columns.",
|
||
op=op,
|
||
)
|
||
)
|
||
number = utils.cast_to_num(eq) # type: ignore[arg-type]
|
||
if number is None:
|
||
raise QueryObjectValidationError(
|
||
_("The Length filter requires a numeric value.")
|
||
)
|
||
length_expr = db_engine_spec.array_length(sqla_col)
|
||
length_comparisons = {
|
||
utils.FilterOperator.LENGTH_EQUALS: length_expr == number,
|
||
utils.FilterOperator.LENGTH_GREATER_THAN: length_expr > number,
|
||
utils.FilterOperator.LENGTH_LESS_THAN: length_expr < number,
|
||
utils.FilterOperator.LENGTH_GREATER_THAN_OR_EQUALS: length_expr
|
||
>= number,
|
||
utils.FilterOperator.LENGTH_LESS_THAN_OR_EQUALS: length_expr
|
||
<= number,
|
||
}
|
||
target_clause_list.append(length_comparisons[op])
|
||
elif op == utils.FilterOperator.IS_TRUE:
|
||
target_clause_list.append(
|
||
db_engine_spec.handle_boolean_filter(sqla_col, op, True)
|
||
)
|
||
elif op == utils.FilterOperator.IS_FALSE:
|
||
target_clause_list.append(
|
||
db_engine_spec.handle_boolean_filter(sqla_col, op, False)
|
||
)
|
||
else:
|
||
if (
|
||
op
|
||
not in {
|
||
utils.FilterOperator.EQUALS,
|
||
utils.FilterOperator.NOT_EQUALS,
|
||
}
|
||
and eq is None
|
||
):
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"Must specify a value for filters "
|
||
"with comparison operators"
|
||
)
|
||
)
|
||
if op in {
|
||
utils.FilterOperator.EQUALS,
|
||
utils.FilterOperator.NOT_EQUALS,
|
||
utils.FilterOperator.GREATER_THAN,
|
||
utils.FilterOperator.LESS_THAN,
|
||
utils.FilterOperator.GREATER_THAN_OR_EQUALS,
|
||
utils.FilterOperator.LESS_THAN_OR_EQUALS,
|
||
}:
|
||
target_clause_list.append(
|
||
db_engine_spec.handle_comparison_filter(sqla_col, op, eq)
|
||
)
|
||
elif op in {
|
||
utils.FilterOperator.ILIKE,
|
||
utils.FilterOperator.LIKE,
|
||
utils.FilterOperator.NOT_LIKE,
|
||
utils.FilterOperator.NOT_ILIKE,
|
||
}:
|
||
# Native UUID columns report GenericDataType.STRING but
|
||
# reject LIKE/ILIKE without a cast (see issue #41795)
|
||
needs_string_cast_for_like: bool = (
|
||
target_generic_type != GenericDataType.STRING
|
||
or is_uuid_native_type(col_type)
|
||
)
|
||
if needs_string_cast_for_like:
|
||
sqla_col = sa.cast(sqla_col, sa.String)
|
||
|
||
if op == utils.FilterOperator.LIKE:
|
||
target_clause_list.append(sqla_col.like(eq))
|
||
elif op == utils.FilterOperator.ILIKE:
|
||
target_clause_list.append(sqla_col.ilike(eq))
|
||
elif op == utils.FilterOperator.NOT_LIKE:
|
||
target_clause_list.append(sqla_col.not_like(eq))
|
||
else:
|
||
target_clause_list.append(sqla_col.not_ilike(eq))
|
||
elif op in {
|
||
utils.FilterOperator.CONTAINS_ANY,
|
||
utils.FilterOperator.CONTAINS_ALL,
|
||
}:
|
||
# Element-level array membership. Enforce the target is
|
||
# actually a multi-value (array) column (only classified
|
||
# MULTI_VALUE on an array-capable engine), guarding against
|
||
# payloads that bypass the UI gating.
|
||
if target_generic_type != GenericDataType.MULTI_VALUE:
|
||
raise QueryObjectValidationError(
|
||
_(
|
||
"The %(op)s operator is only supported for "
|
||
"multi-value (array) columns.",
|
||
op=op,
|
||
)
|
||
)
|
||
array_values: list[Any] = coerce_array_values(
|
||
list(eq) if isinstance(eq, (list, tuple)) else [eq],
|
||
array_element_type,
|
||
)
|
||
if op == utils.FilterOperator.CONTAINS_ANY:
|
||
target_clause_list.append(
|
||
db_engine_spec.array_contains_any(
|
||
sqla_col, array_values
|
||
)
|
||
)
|
||
else:
|
||
target_clause_list.append(
|
||
db_engine_spec.array_contains_all(
|
||
sqla_col, array_values
|
||
)
|
||
)
|
||
elif (
|
||
op == utils.FilterOperator.TEMPORAL_RANGE
|
||
and isinstance(eq, str)
|
||
and col_obj is not None
|
||
):
|
||
_since, _until = get_since_until_from_time_range(
|
||
time_range=eq,
|
||
time_shift=time_shift,
|
||
extras=extras,
|
||
)
|
||
_temporal_filter = self.get_time_filter(
|
||
time_col=col_obj,
|
||
start_dttm=_since,
|
||
end_dttm=_until,
|
||
time_grain=flt_grain,
|
||
label=sqla_col.key,
|
||
template_processor=template_processor,
|
||
)
|
||
if _temporal_filter is not None:
|
||
target_clause_list.append(_temporal_filter)
|
||
else:
|
||
raise QueryObjectValidationError(
|
||
_("Invalid filter operation type: %(op)s", op=op)
|
||
)
|
||
else:
|
||
# col_obj is None and sqla_col is None - column not found!
|
||
# Silently skip - this can happen for removed columns or invalid filters
|
||
pass
|
||
where_clause_and += self.get_sqla_row_level_filters(template_processor)
|
||
if extras:
|
||
where = extras.get("where")
|
||
if where:
|
||
where = self._process_select_expression(
|
||
expression=where,
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
where_clause_and += [Grouping(self.text(where))]
|
||
having = extras.get("having")
|
||
if having:
|
||
having = self._process_select_expression(
|
||
expression=having,
|
||
database_id=self.database_id,
|
||
engine=self.database.backend,
|
||
schema=self.schema,
|
||
template_processor=template_processor,
|
||
)
|
||
having_clause_and += [Grouping(self.text(having))]
|
||
|
||
if apply_fetch_values_predicate and self.fetch_values_predicate:
|
||
qry = qry.where(
|
||
self.get_fetch_values_predicate(template_processor=template_processor)
|
||
)
|
||
|
||
if granularity:
|
||
if time_filters or where_clause_and:
|
||
qry = qry.where(and_(*(time_filters + where_clause_and)))
|
||
elif where_clause_and:
|
||
qry = qry.where(and_(*where_clause_and))
|
||
if having_clause_and:
|
||
qry = qry.having(and_(*having_clause_and))
|
||
|
||
self.make_orderby_compatible(select_exprs, orderby_exprs)
|
||
|
||
for col, (_orig_col, ascending) in zip(orderby_exprs, orderby, strict=False): # noqa: B007
|
||
if not db_engine_spec.allows_alias_in_orderby and isinstance(col, Label):
|
||
# if engine does not allow using SELECT alias in ORDER BY
|
||
# revert to the underlying column
|
||
col = col.element
|
||
|
||
if (
|
||
db_engine_spec.get_allows_alias_in_select(self.database)
|
||
and db_engine_spec.allows_hidden_cc_in_orderby
|
||
and col.name in [select_col.name for select_col in select_exprs]
|
||
):
|
||
col = literal_column(quote(col.name))
|
||
direction = sa.asc if ascending else sa.desc
|
||
qry = qry.order_by(direction(col))
|
||
|
||
# A GROUPING SETS query computes a bounded set of rollup levels; applying
|
||
# a row LIMIT is both unnecessary and unparseable by some SQL parsers
|
||
# (LIMIT after a GROUPING SETS GROUP BY), so skip it for that path.
|
||
if row_limit and not use_grouping_sets:
|
||
qry = qry.limit(row_limit)
|
||
if row_offset and self.database.db_engine_spec.supports_offset:
|
||
qry = qry.offset(row_offset)
|
||
|
||
if series_limit and groupby_series_columns:
|
||
if db_engine_spec.allows_joins and db_engine_spec.allows_subqueries:
|
||
# some sql dialects require for order by expressions
|
||
# to also be in the select clause -- others, e.g. vertica,
|
||
# require a unique inner alias
|
||
inner_main_metric_expr = self.make_sqla_column_compatible(
|
||
main_metric_expr, "mme_inner__"
|
||
)
|
||
inner_groupby_exprs = []
|
||
inner_select_exprs = []
|
||
for gby_name, gby_obj in groupby_series_columns.items():
|
||
inner = self.make_sqla_column_compatible(gby_obj, gby_name + "__")
|
||
inner_groupby_exprs.append(inner)
|
||
inner_select_exprs.append(inner)
|
||
|
||
inner_select_exprs += [inner_main_metric_expr]
|
||
subq = sa.select(*inner_select_exprs).select_from(tbl)
|
||
inner_time_filter = []
|
||
|
||
if dttm_col and not db_engine_spec.time_groupby_inline:
|
||
_inner_filter = self.get_time_filter(
|
||
time_col=dttm_col,
|
||
start_dttm=inner_from_dttm or from_dttm,
|
||
end_dttm=inner_to_dttm or to_dttm,
|
||
template_processor=template_processor,
|
||
)
|
||
if _inner_filter is not None:
|
||
inner_time_filter = [_inner_filter]
|
||
subq = subq.where(and_(*(where_clause_and + inner_time_filter)))
|
||
subq = subq.group_by(*inner_groupby_exprs)
|
||
|
||
ob = inner_main_metric_expr
|
||
if series_limit_metric:
|
||
ob = self._get_series_orderby(
|
||
series_limit_metric=series_limit_metric,
|
||
metrics_by_name=metrics_by_name,
|
||
columns_by_name=columns_by_name,
|
||
template_processor=template_processor,
|
||
)
|
||
direction = sa.desc if order_desc else sa.asc
|
||
subq = subq.order_by(direction(ob))
|
||
subq = subq.limit(series_limit)
|
||
|
||
on_clause = []
|
||
for gby_name, gby_obj in groupby_series_columns.items():
|
||
# in this case the column name, not the alias, needs to be
|
||
# conditionally mutated, as it refers to the column alias in
|
||
# the inner query
|
||
col_name = db_engine_spec.make_label_compatible(gby_name + "__")
|
||
on_clause.append(gby_obj == sa.column(col_name))
|
||
|
||
# Use LEFT JOIN when grouping others, INNER JOIN otherwise
|
||
if group_others_when_limit_reached:
|
||
# Create the alias once and reuse it
|
||
subq_alias = subq.alias(SERIES_LIMIT_SUBQ_ALIAS)
|
||
tbl = tbl.join(
|
||
subq_alias,
|
||
and_(*on_clause),
|
||
isouter=True,
|
||
)
|
||
|
||
# Apply Others grouping using the refactored method
|
||
def _create_join_condition(col_name: str, expr: Any) -> Any:
|
||
# Get the corresponding column from the subquery
|
||
subq_col_name = db_engine_spec.make_label_compatible(
|
||
col_name + "__"
|
||
)
|
||
# Reference the column from the already-created aliased subquery
|
||
subq_col = subq_alias.c[subq_col_name]
|
||
return subq_col.is_not(None)
|
||
|
||
select_exprs, groupby_all_columns = (
|
||
self._apply_series_others_grouping(
|
||
select_exprs,
|
||
groupby_all_columns,
|
||
groupby_series_columns,
|
||
_create_join_condition,
|
||
)
|
||
)
|
||
|
||
# Reconstruct query with modified expressions
|
||
qry = sa.select(*select_exprs)
|
||
if groupby_all_columns:
|
||
qry = qry.group_by(*groupby_all_columns.values())
|
||
|
||
# Re-apply WHERE and HAVING clauses lost during query reconstruction
|
||
qry = self._reapply_query_filters(
|
||
qry,
|
||
apply_fetch_values_predicate,
|
||
template_processor,
|
||
granularity,
|
||
time_filters,
|
||
where_clause_and,
|
||
having_clause_and,
|
||
)
|
||
else:
|
||
tbl = tbl.join(
|
||
subq.alias(SERIES_LIMIT_SUBQ_ALIAS), and_(*on_clause)
|
||
)
|
||
else:
|
||
if series_limit_metric:
|
||
orderby = [
|
||
(
|
||
self._get_series_orderby(
|
||
series_limit_metric=series_limit_metric,
|
||
metrics_by_name=metrics_by_name,
|
||
columns_by_name=columns_by_name,
|
||
template_processor=template_processor,
|
||
),
|
||
not order_desc,
|
||
)
|
||
]
|
||
|
||
# run prequery to get top groups
|
||
prequery_obj: QueryObjectDict = {
|
||
"is_timeseries": False,
|
||
"row_limit": series_limit,
|
||
"metrics": metrics,
|
||
"granularity": granularity,
|
||
"groupby": groupby,
|
||
"from_dttm": inner_from_dttm or from_dttm,
|
||
"to_dttm": inner_to_dttm or to_dttm,
|
||
"filter": filter or [],
|
||
"orderby": orderby,
|
||
"extras": extras,
|
||
"columns": get_non_base_axis_columns(columns),
|
||
"order_desc": True,
|
||
}
|
||
|
||
result = self.query(prequery_obj)
|
||
prequeries.append(result.query)
|
||
dimensions = [
|
||
c
|
||
for c in result.df.columns
|
||
if c not in metrics and c in groupby_series_columns
|
||
]
|
||
top_groups = self._get_top_groups(
|
||
result.df, dimensions, groupby_series_columns, columns_by_name
|
||
)
|
||
|
||
if group_others_when_limit_reached:
|
||
# Apply Others grouping using the refactored method
|
||
def _create_top_groups_condition(col_name: str, expr: Any) -> Any:
|
||
return top_groups
|
||
|
||
select_exprs, groupby_all_columns = (
|
||
self._apply_series_others_grouping(
|
||
select_exprs,
|
||
groupby_all_columns,
|
||
groupby_series_columns,
|
||
_create_top_groups_condition,
|
||
)
|
||
)
|
||
|
||
# Reconstruct query with modified expressions
|
||
qry = sa.select(*select_exprs)
|
||
if groupby_all_columns:
|
||
qry = qry.group_by(*groupby_all_columns.values())
|
||
|
||
# Re-apply WHERE and HAVING clauses lost during query reconstruction
|
||
qry = self._reapply_query_filters(
|
||
qry,
|
||
apply_fetch_values_predicate,
|
||
template_processor,
|
||
granularity,
|
||
time_filters,
|
||
where_clause_and,
|
||
having_clause_and,
|
||
)
|
||
else:
|
||
# Original behavior: filter to only top groups
|
||
qry = qry.where(top_groups)
|
||
|
||
qry = qry.select_from(tbl)
|
||
|
||
if is_rowcount:
|
||
if not db_engine_spec.allows_subqueries:
|
||
raise QueryObjectValidationError(
|
||
_("Database does not support subqueries")
|
||
)
|
||
label = "rowcount"
|
||
col = self.make_sqla_column_compatible(literal_column("COUNT(*)"), label)
|
||
qry = sa.select(col).select_from(qry.alias("rowcount_qry"))
|
||
labels_expected = [label]
|
||
|
||
filter_columns = [flt.get("col") for flt in filter] if filter else []
|
||
rejected_filter_columns = [
|
||
col
|
||
for col in filter_columns
|
||
if col
|
||
and not is_adhoc_column(col)
|
||
and col not in self.column_names
|
||
and col not in applied_template_filters
|
||
] + rejected_adhoc_filters_columns
|
||
|
||
applied_filter_columns = [
|
||
col
|
||
for col in filter_columns
|
||
if col
|
||
and not is_adhoc_column(col)
|
||
and (col in self.column_names or col in applied_template_filters)
|
||
] + applied_adhoc_filters_columns
|
||
|
||
return SqlaQuery(
|
||
applied_template_filters=applied_template_filters,
|
||
cte=cte,
|
||
applied_filter_columns=applied_filter_columns,
|
||
rejected_filter_columns=rejected_filter_columns,
|
||
extra_cache_keys=extra_cache_keys,
|
||
labels_expected=labels_expected,
|
||
sqla_query=qry,
|
||
prequeries=prequeries,
|
||
sql_shifted_temporal_labels=sql_shifted_temporal_labels,
|
||
)
|