mirror of
https://github.com/apache/superset.git
synced 2026-09-01 21:11:28 +00:00
666 lines
28 KiB
Python
666 lines
28 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
# pylint: disable=invalid-name
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
import logging
|
|
from datetime import datetime
|
|
from pprint import pformat
|
|
from typing import Any, NamedTuple, TYPE_CHECKING
|
|
|
|
from flask import current_app
|
|
from flask_babel import gettext as _
|
|
from jinja2.exceptions import TemplateError
|
|
from pandas import DataFrame
|
|
|
|
from superset import feature_flag_manager
|
|
from superset.common.chart_data import ChartDataResultType
|
|
from superset.exceptions import (
|
|
InvalidPostProcessingError,
|
|
QueryClauseValidationException,
|
|
QueryObjectValidationError,
|
|
)
|
|
from superset.extensions import event_logger
|
|
from superset.sql.parse import sanitize_clause, transpile_to_dialect
|
|
from superset.superset_typing import Column, Metric, OrderBy, QueryObjectDict
|
|
from superset.utils import json, pandas_postprocessing
|
|
from superset.utils.cache_keys import add_impersonation_cache_key_if_needed
|
|
from superset.utils.core import (
|
|
DTTM_ALIAS,
|
|
find_duplicates,
|
|
get_column_names,
|
|
get_metric_names,
|
|
is_adhoc_metric,
|
|
QueryObjectFilterClause,
|
|
)
|
|
from superset.utils.hashing import hash_from_dict
|
|
from superset.utils.json import json_int_dttm_ser
|
|
|
|
if TYPE_CHECKING:
|
|
from superset.connectors.sqla.models import BaseDatasource
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# TODO: Type Metrics dictionary with TypedDict when it becomes a vanilla python type
|
|
# https://github.com/python/mypy/issues/5288
|
|
|
|
|
|
class DeprecatedField(NamedTuple):
|
|
old_name: str
|
|
new_name: str
|
|
|
|
|
|
DEPRECATED_FIELDS = (
|
|
DeprecatedField(old_name="granularity_sqla", new_name="granularity"),
|
|
DeprecatedField(old_name="groupby", new_name="columns"),
|
|
DeprecatedField(old_name="timeseries_limit", new_name="series_limit"),
|
|
DeprecatedField(old_name="timeseries_limit_metric", new_name="series_limit_metric"),
|
|
)
|
|
|
|
DEPRECATED_EXTRAS_FIELDS = (
|
|
DeprecatedField(old_name="where", new_name="where"),
|
|
DeprecatedField(old_name="having", new_name="having"),
|
|
)
|
|
|
|
|
|
class QueryObject: # pylint: disable=too-many-instance-attributes
|
|
"""
|
|
The query objects are constructed on the client.
|
|
"""
|
|
|
|
annotation_layers: list[dict[str, Any]]
|
|
applied_time_extras: dict[str, str]
|
|
apply_fetch_values_predicate: bool
|
|
columns: list[Column]
|
|
datasource: BaseDatasource | None
|
|
extras: dict[str, Any]
|
|
filter: list[QueryObjectFilterClause]
|
|
from_dttm: datetime | None
|
|
granularity: str | None
|
|
grouping_sets: list[list[str]]
|
|
inner_from_dttm: datetime | None
|
|
inner_to_dttm: datetime | None
|
|
is_rowcount: bool
|
|
is_timeseries: bool
|
|
metrics: list[Metric] | None
|
|
order_desc: bool
|
|
orderby: list[OrderBy]
|
|
post_processing: list[dict[str, Any]]
|
|
result_type: ChartDataResultType | None
|
|
row_limit: int | None
|
|
row_offset: int
|
|
series_columns: list[Column]
|
|
series_limit: int
|
|
series_limit_metric: Metric | None
|
|
time_offsets: list[str]
|
|
time_compare_full_range: bool
|
|
time_shift: str | None
|
|
time_range: str | None
|
|
to_dttm: datetime | None
|
|
|
|
def __init__( # pylint: disable=too-many-locals, too-many-arguments
|
|
self,
|
|
*,
|
|
annotation_layers: list[dict[str, Any]] | None = None,
|
|
applied_time_extras: dict[str, str] | None = None,
|
|
apply_fetch_values_predicate: bool = False,
|
|
columns: list[Column] | None = None,
|
|
datasource: BaseDatasource | None = None,
|
|
extras: dict[str, Any] | None = None,
|
|
filters: list[QueryObjectFilterClause] | None = None,
|
|
granularity: str | None = None,
|
|
is_rowcount: bool = False,
|
|
is_timeseries: bool | None = None,
|
|
metrics: list[Metric] | None = None,
|
|
order_desc: bool = True,
|
|
orderby: list[OrderBy] | None = None,
|
|
post_processing: list[dict[str, Any] | None] | None = None,
|
|
row_limit: int | None = None,
|
|
row_offset: int | None = None,
|
|
series_columns: list[Column] | None = None,
|
|
series_limit: int = 0,
|
|
series_limit_metric: Metric | None = None,
|
|
group_others_when_limit_reached: bool = False,
|
|
grouping_sets: list[list[str]] | None = None,
|
|
time_range: str | None = None,
|
|
time_shift: str | None = None,
|
|
**kwargs: Any,
|
|
):
|
|
self._set_annotation_layers(annotation_layers)
|
|
self.applied_time_extras = applied_time_extras or {}
|
|
self.apply_fetch_values_predicate = apply_fetch_values_predicate or False
|
|
self.columns = columns or []
|
|
self.datasource = datasource
|
|
self.extras = extras or {}
|
|
self.filter = filters or []
|
|
self.granularity = granularity
|
|
self.is_rowcount = is_rowcount
|
|
self._set_is_timeseries(is_timeseries)
|
|
self._set_metrics(metrics)
|
|
self.order_desc = order_desc
|
|
self.orderby = orderby or []
|
|
self._set_post_processing(post_processing)
|
|
self.row_limit = row_limit
|
|
self.row_offset = row_offset or 0
|
|
self._init_series_columns(series_columns, metrics, is_timeseries)
|
|
self.series_limit = series_limit
|
|
self.series_limit_metric = series_limit_metric
|
|
self.group_others_when_limit_reached = group_others_when_limit_reached
|
|
self.grouping_sets = grouping_sets or []
|
|
self.time_range = time_range
|
|
self.time_shift = time_shift
|
|
self.from_dttm = kwargs.get("from_dttm")
|
|
self.to_dttm = kwargs.get("to_dttm")
|
|
self.result_type = kwargs.get("result_type")
|
|
self.time_offsets = kwargs.get("time_offsets", [])
|
|
self.time_compare_full_range = kwargs.get("time_compare_full_range", False)
|
|
self.inner_from_dttm = kwargs.get("inner_from_dttm")
|
|
self.inner_to_dttm = kwargs.get("inner_to_dttm")
|
|
self._rename_deprecated_fields(kwargs)
|
|
self._move_deprecated_extra_fields(kwargs)
|
|
|
|
def _set_annotation_layers(
|
|
self, annotation_layers: list[dict[str, Any]] | None
|
|
) -> None:
|
|
self.annotation_layers = [
|
|
layer
|
|
for layer in (annotation_layers or [])
|
|
# formula annotations don't affect the payload, hence can be dropped
|
|
if layer["annotationType"] != "FORMULA"
|
|
]
|
|
|
|
def _set_is_timeseries(self, is_timeseries: bool | None) -> None:
|
|
# is_timeseries is True if time column is in either columns or groupby
|
|
# (both are dimensions)
|
|
self.is_timeseries = (
|
|
is_timeseries if is_timeseries is not None else DTTM_ALIAS in self.columns
|
|
)
|
|
|
|
def _set_metrics(self, metrics: list[Metric] | None = None) -> None:
|
|
# Support metric reference/definition in the format of
|
|
# 1. 'metric_name' - name of predefined metric
|
|
# 2. { label: 'label_name' } - legacy format for a predefined metric
|
|
# 3. { expressionType: 'SIMPLE' | 'SQL', ... } - adhoc metric
|
|
# Keys that only ever appear on an ad-hoc metric definition. A dict
|
|
# carrying one of these but missing `expressionType` is a malformed
|
|
# ad-hoc metric, not a legacy predefined-metric reference, and must
|
|
# not be silently collapsed to its label, which would later be
|
|
# misread as a request for a saved metric of that name.
|
|
adhoc_metric_keys = {"sqlExpression", "aggregate", "column"}
|
|
|
|
def normalize_metric(metric: Metric) -> Metric:
|
|
if isinstance(metric, str) or is_adhoc_metric(metric):
|
|
return metric
|
|
if adhoc_metric_keys & metric.keys():
|
|
raise QueryObjectValidationError(
|
|
_(
|
|
"Invalid ad-hoc metric %(label)s: `expressionType` is missing",
|
|
label=metric.get("label"),
|
|
)
|
|
)
|
|
return metric["label"] # type: ignore
|
|
|
|
self.metrics = metrics and [normalize_metric(x) for x in metrics]
|
|
|
|
def _set_post_processing(
|
|
self, post_processing: list[dict[str, Any] | None] | None
|
|
) -> None:
|
|
self.post_processing = [
|
|
self._drop_unsupported_options(post_proc)
|
|
for post_proc in post_processing or []
|
|
if post_proc
|
|
]
|
|
|
|
@staticmethod
|
|
def _drop_unsupported_options(post_proc: dict[str, Any]) -> dict[str, Any]:
|
|
"""
|
|
Drop options that the post-processing operation no longer accepts.
|
|
|
|
A chart's ``query_context`` is written when the chart is saved and is
|
|
never rewritten afterwards, while Explore rebuilds the query from
|
|
``form_data`` at every render. A chart saved by an older version of
|
|
Superset can therefore reference an option that has since been removed
|
|
from the operation. ``exec_post_processing`` passes the stored options
|
|
as keyword arguments, so that option raises a bare ``TypeError`` on
|
|
every path that replays the stored ``query_context`` -- the chart data
|
|
endpoint, alerts and reports, thumbnails, CSV export -- while the same
|
|
chart still renders correctly in Explore.
|
|
|
|
Comparing against the signature avoids a hard-coded list of removed
|
|
option names, which would need extending at each release.
|
|
|
|
Only the built-in operations in ``pandas_postprocessing.__all__`` are
|
|
inspected. The module also exposes helpers, imported submodules and
|
|
typing aliases, none of which are operations; and options belonging to a
|
|
callable registered through ``EXTRA_PANDAS_POSTPROCESSING_OPS`` are the
|
|
operator's to manage, so both are passed through untouched.
|
|
"""
|
|
operation = post_proc.get("operation")
|
|
function = (
|
|
getattr(pandas_postprocessing, operation, None)
|
|
if isinstance(operation, str) and operation in pandas_postprocessing.__all__
|
|
else None
|
|
)
|
|
if function is None:
|
|
# A missing, unknown or operator-registered operation is left
|
|
# untouched, so that exec_post_processing either dispatches it or
|
|
# reports it as InvalidPostProcessingError.
|
|
return post_proc
|
|
|
|
parameters = inspect.signature(function).parameters
|
|
if any(
|
|
parameter.kind is inspect.Parameter.VAR_KEYWORD
|
|
for parameter in parameters.values()
|
|
):
|
|
return post_proc
|
|
|
|
# `exec_post_processing` calls the operation as `operation(df, **options)`,
|
|
# so an option can only reach a parameter that a caller may fill by
|
|
# keyword. That excludes the first parameter, which receives the
|
|
# DataFrame positionally, and any positional-only or `*args` parameter.
|
|
keyword_parameters = {
|
|
name
|
|
for position, (name, parameter) in enumerate(parameters.items())
|
|
if position > 0
|
|
and parameter.kind
|
|
in (
|
|
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
inspect.Parameter.KEYWORD_ONLY,
|
|
)
|
|
}
|
|
|
|
options = post_proc.get("options") or {}
|
|
unsupported = {key for key in options if key not in keyword_parameters}
|
|
if not unsupported:
|
|
return post_proc
|
|
|
|
# Logged at info: a chart saved before the option was removed hits this
|
|
# on every render, so a warning would repeat for as long as the chart
|
|
# is not resaved, without anything new to report.
|
|
logger.info(
|
|
"Dropping unsupported option(s) %s of post-processing operation "
|
|
"`%s`. The chart's stored query_context predates the current "
|
|
"signature of that operation.",
|
|
sorted(unsupported),
|
|
operation,
|
|
)
|
|
return {
|
|
**post_proc,
|
|
"options": {
|
|
key: value
|
|
for key, value in options.items()
|
|
if key in keyword_parameters
|
|
},
|
|
}
|
|
|
|
def _init_series_columns(
|
|
self,
|
|
series_columns: list[Column] | None,
|
|
metrics: list[Metric] | None,
|
|
is_timeseries: bool | None,
|
|
) -> None:
|
|
if series_columns:
|
|
self.series_columns = series_columns
|
|
elif is_timeseries and metrics:
|
|
self.series_columns = self.columns
|
|
else:
|
|
self.series_columns = []
|
|
|
|
def _rename_deprecated_fields(self, kwargs: dict[str, Any]) -> None:
|
|
# rename deprecated fields
|
|
for field in DEPRECATED_FIELDS:
|
|
if field.old_name in kwargs:
|
|
logger.warning(
|
|
"The field `%s` is deprecated, please use `%s` instead.",
|
|
field.old_name,
|
|
field.new_name,
|
|
)
|
|
value = kwargs[field.old_name]
|
|
if value:
|
|
if hasattr(self, field.new_name):
|
|
logger.warning(
|
|
"The field `%s` is already populated, "
|
|
"replacing value with contents from `%s`.",
|
|
field.new_name,
|
|
field.old_name,
|
|
)
|
|
setattr(self, field.new_name, value)
|
|
|
|
def _move_deprecated_extra_fields(self, kwargs: dict[str, Any]) -> None:
|
|
# move deprecated extras fields to extras
|
|
for field in DEPRECATED_EXTRAS_FIELDS:
|
|
if field.old_name in kwargs:
|
|
logger.warning(
|
|
"The field `%s` is deprecated and should "
|
|
"be passed to `extras` via the `%s` property.",
|
|
field.old_name,
|
|
field.new_name,
|
|
)
|
|
value = kwargs[field.old_name]
|
|
if value:
|
|
if hasattr(self.extras, field.new_name):
|
|
logger.warning(
|
|
"The field `%s` is already populated in "
|
|
"`extras`, replacing value with contents "
|
|
"from `%s`.",
|
|
field.new_name,
|
|
field.old_name,
|
|
)
|
|
self.extras[field.new_name] = value
|
|
|
|
@property
|
|
def metric_names(self) -> list[str]:
|
|
"""Return metrics names (labels), coerce adhoc metrics to strings."""
|
|
return get_metric_names(
|
|
self.metrics or [],
|
|
(
|
|
self.datasource.verbose_map
|
|
if self.datasource and hasattr(self.datasource, "verbose_map")
|
|
else None
|
|
),
|
|
)
|
|
|
|
@property
|
|
def column_names(self) -> list[str]:
|
|
"""Return column names (labels). Gives priority to groupbys if both groupbys
|
|
and metrics are non-empty, otherwise returns column labels."""
|
|
return get_column_names(self.columns)
|
|
|
|
def validate(
|
|
self, raise_exceptions: bool | None = True
|
|
) -> QueryObjectValidationError | None:
|
|
"""Validate query object"""
|
|
try:
|
|
self._validate_there_are_no_missing_series()
|
|
self._validate_no_have_duplicate_labels()
|
|
self._validate_time_offsets()
|
|
self._sanitize_filters()
|
|
return None
|
|
except QueryObjectValidationError as ex:
|
|
if raise_exceptions:
|
|
raise
|
|
return ex
|
|
|
|
def _validate_no_have_duplicate_labels(self) -> None:
|
|
all_labels = self.metric_names + self.column_names
|
|
if len(set(all_labels)) < len(all_labels):
|
|
dup_labels = find_duplicates(all_labels)
|
|
raise QueryObjectValidationError(
|
|
_(
|
|
"Duplicate column/metric labels: %(labels)s. Please make "
|
|
"sure all columns and metrics have a unique label.",
|
|
labels=", ".join(f'"{x}"' for x in dup_labels),
|
|
)
|
|
)
|
|
|
|
def _validate_time_offsets(self) -> None:
|
|
"""Validate time_offsets configuration"""
|
|
if not self.time_offsets:
|
|
return
|
|
|
|
for offset in self.time_offsets:
|
|
# Check if this is a date range offset (YYYY-MM-DD : YYYY-MM-DD format)
|
|
if self._is_valid_date_range(offset):
|
|
if not feature_flag_manager.is_feature_enabled(
|
|
"DATE_RANGE_TIMESHIFTS_ENABLED"
|
|
):
|
|
raise QueryObjectValidationError(
|
|
"Date range timeshifts are not enabled. "
|
|
"Please contact your administrator to enable the "
|
|
"DATE_RANGE_TIMESHIFTS_ENABLED feature flag."
|
|
)
|
|
|
|
def _is_valid_date_range(self, date_range: str) -> bool:
|
|
"""Check if string is a valid date range in YYYY-MM-DD : YYYY-MM-DD format"""
|
|
try:
|
|
# Attempt to parse the string as a date range in the format
|
|
# YYYY-MM-DD:YYYY-MM-DD
|
|
start_date, end_date = date_range.split(":")
|
|
datetime.strptime(start_date.strip(), "%Y-%m-%d")
|
|
datetime.strptime(end_date.strip(), "%Y-%m-%d")
|
|
return True
|
|
except ValueError:
|
|
# If parsing fails, it's not a valid date range in the format
|
|
# YYYY-MM-DD:YYYY-MM-DD
|
|
return False
|
|
|
|
def _sanitize_filters(self) -> None:
|
|
from superset.jinja_context import get_template_processor
|
|
|
|
needs_transpilation = self.extras.get("transpile_to_dialect", False)
|
|
|
|
for param in ("where", "having"):
|
|
clause = self.extras.get(param)
|
|
if clause and self.datasource:
|
|
try:
|
|
database = self.datasource.database
|
|
processor = get_template_processor(
|
|
database=database, table=self.datasource
|
|
)
|
|
try:
|
|
clause = processor.process_template(clause, force=True)
|
|
except TemplateError as ex:
|
|
raise QueryObjectValidationError(
|
|
_(
|
|
"Error in jinja expression in WHERE clause: %(msg)s",
|
|
msg=ex.message,
|
|
)
|
|
) from ex
|
|
|
|
engine = database.db_engine_spec.engine
|
|
|
|
if needs_transpilation:
|
|
# source_engine=engine ensures idempotency: this
|
|
# method can run more than once (validate() is called
|
|
# from both raise_for_access and get_df_payload), so
|
|
# the second pass must be able to re-parse the
|
|
# dialect-specific output (e.g. BigQuery backticks)
|
|
# produced by the first pass.
|
|
clause = transpile_to_dialect(
|
|
clause, engine, source_engine=engine, identify=True
|
|
)
|
|
|
|
sanitized_clause = sanitize_clause(clause, engine)
|
|
self.extras[param] = sanitized_clause
|
|
except QueryClauseValidationException as ex:
|
|
raise QueryObjectValidationError(ex.message) from ex
|
|
|
|
def _validate_there_are_no_missing_series(self) -> None:
|
|
missing_series = [col for col in self.series_columns if col not in self.columns]
|
|
if missing_series:
|
|
raise QueryObjectValidationError(
|
|
_(
|
|
"The following entries in `series_columns` are missing "
|
|
"in `columns`: %(columns)s. ",
|
|
columns=", ".join(f'"{x}"' for x in missing_series),
|
|
)
|
|
)
|
|
|
|
def to_dict(self) -> QueryObjectDict:
|
|
query_object_dict: QueryObjectDict = {
|
|
"apply_fetch_values_predicate": self.apply_fetch_values_predicate,
|
|
"columns": self.columns,
|
|
"extras": self.extras,
|
|
"filter": self.filter,
|
|
"from_dttm": self.from_dttm,
|
|
"granularity": self.granularity,
|
|
"inner_from_dttm": self.inner_from_dttm,
|
|
"inner_to_dttm": self.inner_to_dttm,
|
|
"is_rowcount": self.is_rowcount,
|
|
"is_timeseries": self.is_timeseries,
|
|
"metrics": self.metrics,
|
|
"order_desc": self.order_desc,
|
|
"orderby": self.orderby,
|
|
"post_processing": self.post_processing,
|
|
"row_limit": self.row_limit,
|
|
"row_offset": self.row_offset,
|
|
"series_columns": self.series_columns,
|
|
"series_limit": self.series_limit,
|
|
"series_limit_metric": self.series_limit_metric,
|
|
"group_others_when_limit_reached": self.group_others_when_limit_reached,
|
|
"grouping_sets": self.grouping_sets,
|
|
"to_dttm": self.to_dttm,
|
|
"time_shift": self.time_shift,
|
|
"time_compare_full_range": self.time_compare_full_range,
|
|
}
|
|
return query_object_dict
|
|
|
|
def __repr__(self) -> str:
|
|
# we use `print` or `logging` output QueryObject
|
|
return json.dumps(
|
|
self.to_dict(),
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
|
|
def cache_key(self, **extra: Any) -> str: # noqa: C901
|
|
"""
|
|
The cache key is made out of the key/values from to_dict(), plus any
|
|
other key/values in `extra`
|
|
We remove datetime bounds that are hard values, and replace them with
|
|
the use-provided inputs to bounds, which may be time-relative (as in
|
|
"5 days ago" or "now").
|
|
"""
|
|
# Cast to dict[str, Any] for mutation operations
|
|
cache_dict: dict[str, Any] = dict(self.to_dict())
|
|
cache_dict.update(extra)
|
|
|
|
if "extra_cache_keys" in cache_dict:
|
|
# Order carries no meaning here (an unordered set of opaque
|
|
# Jinja url_param()-derived values), but hash_from_dict only
|
|
# sorts dict keys, not list values, so an unsorted list makes
|
|
# the cache key depend on Python's per-process hash-randomized
|
|
# set iteration order (see SqlaTable.get_extra_cache_keys).
|
|
# Normalize once here so every producer of extra_cache_keys is
|
|
# safe by construction. Sort on (type name, str value) rather
|
|
# than a bare str() so values that stringify identically but
|
|
# differ in type (e.g. 1 and "1") still sort deterministically
|
|
# instead of falling back to input order.
|
|
cache_dict["extra_cache_keys"] = sorted(
|
|
cache_dict["extra_cache_keys"],
|
|
key=lambda value: (type(value).__name__, str(value)),
|
|
)
|
|
|
|
# TODO: the below KVs can all be cleaned up and moved to `to_dict()` at some
|
|
# predetermined point in time when orgs are aware that the previously
|
|
# cached results will be invalidated.
|
|
if not self.apply_fetch_values_predicate:
|
|
del cache_dict["apply_fetch_values_predicate"]
|
|
if self.datasource:
|
|
cache_dict["datasource"] = self.datasource.uid
|
|
if self.result_type:
|
|
cache_dict["result_type"] = self.result_type
|
|
if self.time_range:
|
|
cache_dict["time_range"] = self.time_range
|
|
if self.post_processing:
|
|
# Exclude contribution_totals from post_processing as it's computed at
|
|
# runtime and varies per request, which would cause cache key mismatches
|
|
post_processing_for_cache = []
|
|
for pp in self.post_processing:
|
|
pp_copy = dict(pp)
|
|
if pp_copy.get("operation") == "contribution" and "options" in pp_copy:
|
|
options = dict(pp_copy["options"])
|
|
# Remove contribution_totals as it's dynamically calculated
|
|
options.pop("contribution_totals", None)
|
|
pp_copy["options"] = options
|
|
post_processing_for_cache.append(pp_copy)
|
|
cache_dict["post_processing"] = post_processing_for_cache
|
|
if self.time_offsets:
|
|
cache_dict["time_offsets"] = self.time_offsets
|
|
|
|
for k in ["from_dttm", "to_dttm"]:
|
|
del cache_dict[k]
|
|
|
|
annotation_fields = [
|
|
"annotationType",
|
|
"descriptionColumns",
|
|
"intervalEndColumn",
|
|
"name",
|
|
"overrides",
|
|
"sourceType",
|
|
"timeColumn",
|
|
"titleColumn",
|
|
"value",
|
|
]
|
|
annotation_layers = [
|
|
{field: layer[field] for field in annotation_fields if field in layer}
|
|
for layer in self.annotation_layers
|
|
]
|
|
# only add to key if there are annotations present that affect the payload
|
|
if annotation_layers:
|
|
cache_dict["annotation_layers"] = annotation_layers
|
|
|
|
# Add an impersonation key to cache if impersonation is enabled on the db
|
|
# or if the CACHE_QUERY_BY_USER flag is on or per_user_caching is enabled on
|
|
# the database
|
|
try:
|
|
add_impersonation_cache_key_if_needed(self.datasource.database, cache_dict) # type: ignore
|
|
except AttributeError:
|
|
# datasource or database do not exist
|
|
pass
|
|
|
|
cache_key = hash_from_dict(
|
|
cache_dict, default=json_int_dttm_ser, ignore_nan=True
|
|
)
|
|
# Log QueryObject cache key generation for debugging
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
logger.debug(
|
|
"QueryObject CACHE KEY generated: %s from dict with keys: %s",
|
|
cache_key,
|
|
sorted(cache_dict.keys()),
|
|
)
|
|
return cache_key
|
|
|
|
def exec_post_processing(self, df: DataFrame) -> DataFrame:
|
|
"""
|
|
Perform post processing operations on DataFrame.
|
|
|
|
:param df: DataFrame returned from database model.
|
|
:return: new DataFrame to which all post processing operations have been
|
|
applied
|
|
:raises QueryObjectValidationError: If the post processing operation
|
|
is incorrect
|
|
"""
|
|
logger.debug("post_processing: \n %s", pformat(self.post_processing))
|
|
with event_logger.log_context(f"{self.__class__.__name__}.post_processing"):
|
|
for post_process in self.post_processing:
|
|
operation = post_process.get("operation")
|
|
if not operation:
|
|
raise InvalidPostProcessingError(
|
|
_("`operation` property of post processing object undefined")
|
|
)
|
|
# ``__all__`` is the authoritative list of built-in operations.
|
|
# ``hasattr`` would also match module internals (helpers, imported
|
|
# submodules, typing aliases), shadowing a like-named custom op.
|
|
if operation in pandas_postprocessing.__all__:
|
|
func = getattr(pandas_postprocessing, operation)
|
|
else:
|
|
extra_ops = pandas_postprocessing.build_extra_ops_map(
|
|
current_app.config.get("EXTRA_PANDAS_POSTPROCESSING_OPS", [])
|
|
)
|
|
if operation not in extra_ops:
|
|
raise InvalidPostProcessingError(
|
|
_(
|
|
"Unsupported post processing operation: %(operation)s",
|
|
operation=operation,
|
|
)
|
|
)
|
|
func = extra_ops[operation]
|
|
df = func(df, **post_process.get("options", {}))
|
|
return df
|