# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """Utility functions used across Superset""" # pylint: disable=too-many-lines from __future__ import annotations import _thread import collections import errno import logging import os import platform import re import signal import smtplib import sqlite3 import ssl import tempfile import threading import traceback import uuid import warnings import zlib from collections.abc import Collection, Iterable, Iterator, Sequence from contextlib import closing, contextmanager from dataclasses import dataclass from datetime import timedelta from email.mime.application import MIMEApplication from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate from enum import Enum, IntEnum from io import BytesIO from timeit import default_timer from types import TracebackType from typing import ( Any, Callable, cast, NamedTuple, Optional, TYPE_CHECKING, TypedDict, TypeVar, ) from urllib.parse import unquote_plus, urlparse from zipfile import ZipFile from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import markdown as md import nh3 import pandas as pd import sqlalchemy as sa from cryptography.hazmat.backends import default_backend from cryptography.x509 import Certificate, load_pem_x509_certificate from flask import current_app as app, g, request from flask_appbuilder.security.sqla.models import User from flask_babel import gettext as __ from flask_sqlalchemy import SQLAlchemy from markupsafe import Markup from pandas.api.types import infer_dtype from pandas.core.dtypes.common import is_numeric_dtype from sqlalchemy import event, exc, inspect, select, Text from sqlalchemy.dialects.mysql import LONGTEXT, MEDIUMTEXT from sqlalchemy.engine import Connection, Engine from sqlalchemy.engine.reflection import Inspector from sqlalchemy.sql.type_api import Variant from sqlalchemy.types import TypeEngine from typing_extensions import TypeGuard from superset.constants import ( DEFAULT_USER_AGENT, EXTRA_FORM_DATA_APPEND_KEYS, EXTRA_FORM_DATA_OVERRIDE_EXTRA_KEYS, EXTRA_FORM_DATA_OVERRIDE_REGULAR_MAPPINGS, NO_TIME_RANGE, ) from superset.errors import ErrorLevel, SupersetErrorType from superset.exceptions import ( CertificateException, SupersetException, SupersetTimeoutException, ) from superset.superset_typing import ( AdhocColumn, AdhocMetric, AdhocMetricColumn, Column, FilterValues, FlaskResponse, FormData, Metric, ) from superset.utils.backports import StrEnum from superset.utils.database import get_example_database from superset.utils.date_parser import parse_human_timedelta from superset.utils.hashing import hash_from_dict, hash_from_str from superset.utils.pandas import detect_datetime_format if TYPE_CHECKING: from superset.explorables.base import ColumnMetadata, Explorable from superset.models.core import Database logging.getLogger("MARKDOWN").setLevel(logging.INFO) logger = logging.getLogger(__name__) EMAIL_ATTACHMENT_SUBTYPES: dict[str, str] = { ".pdf": "pdf", ".zip": "zip", ".xlsx": "vnd.openxmlformats-officedocument.spreadsheetml.sheet", } def build_email_attachment(name: str, body: bytes | str) -> MIMEApplication: """ Create an email attachment part with stable filename metadata. """ subtype = EMAIL_ATTACHMENT_SUBTYPES.get(os.path.splitext(name)[1].lower()) payload = body.encode("utf-8") if isinstance(body, str) else body attachment = MIMEApplication( payload, _subtype=subtype or "octet-stream", Name=name, ) attachment.add_header("Content-Disposition", "attachment", filename=name) return attachment DTTM_ALIAS = "__timestamp" TIME_COMPARISON = "__" JS_MAX_INTEGER = 9007199254740991 # Largest int Java Script can handle 2^53-1 InputType = TypeVar("InputType") # pylint: disable=invalid-name ADHOC_FILTERS_REGEX = re.compile("^adhoc_filters") TYPE_MAPPING = { re.compile(r"INT", re.IGNORECASE): "integer", re.compile(r"CHAR|TEXT|VARCHAR", re.IGNORECASE): "string", re.compile(r"DECIMAL|NUMERIC|FLOAT|DOUBLE", re.IGNORECASE): "floating", re.compile(r"BOOL", re.IGNORECASE): "boolean", re.compile(r"DATE|TIME", re.IGNORECASE): "datetime64", } METRIC_MAP_TYPE = { "SUM": "floating", "AVG": "floating", "COUNT": "floating", "COUNT_DISTINCT": "floating", "MIN": "numeric", "MAX": "numeric", "FIRST": "string", "LAST": "string", "GROUP_CONCAT": "string", "ARRAY_AGG": "string", "STRING_AGG": "string", "MEDIAN": "floating", "PERCENTILE": "floating", "VARIANCE": "floating", "STDDEV": "floating", "STDDEV_SAMP": "floating", "VAR_SAMP": "floating", } class AdhocMetricExpressionType(StrEnum): SIMPLE = "SIMPLE" SQL = "SQL" # Aggregates with no safe, universal cross-dialect spelling -- unlike # SUM/COUNT/AVG/MIN/MAX/COUNT_DISTINCT, whose SQL is generated the same way on # every engine. Support for these is opt-in per `BaseEngineSpec` (see # `get_extended_aggregation_func`); used to distinguish a genuinely invalid # aggregate name from one that is valid but unsupported on the current database, # for a clearer user-facing error. EXTENDED_METRIC_AGGREGATES = frozenset({"MEDIAN", "STDDEV_SAMP", "VAR_SAMP"}) class SqlExpressionType(StrEnum): """Types of SQL expressions that can be validated.""" COLUMN = "column" METRIC = "metric" WHERE = "where" HAVING = "having" class AnnotationType(StrEnum): FORMULA = "FORMULA" INTERVAL = "INTERVAL" EVENT = "EVENT" TIME_SERIES = "TIME_SERIES" class GenericDataType(IntEnum): """ Generic database column type that fits both frontend and backend. """ NUMERIC = 0 STRING = 1 TEMPORAL = 2 BOOLEAN = 3 MULTI_VALUE = 4 # array-typed columns (e.g. ClickHouse Array, Postgres ARRAY) # JSON = 5 # and leaving these as a reminder. # MAP = 6 # ROW = 7 class DatasourceType(StrEnum): TABLE = "table" DATASET = "dataset" QUERY = "query" SAVEDQUERY = "saved_query" VIEW = "view" SEMANTIC_VIEW = "semantic_view" class LoggerLevel(StrEnum): INFO = "info" WARNING = "warning" EXCEPTION = "exception" class HeaderDataType(TypedDict): notification_format: str editors: list[int] notification_type: str notification_source: str | None chart_id: int | None dashboard_id: int | None slack_channels: list[str] | None execution_id: str | None class DatasourceDict(TypedDict): type: str # todo(hugh): update this to be DatasourceType id: int | str class AdhocFilterClause(TypedDict, total=False): clause: str expressionType: str filterOptionName: str | None comparator: FilterValues | None operator: str subject: str isExtra: bool | None sqlExpression: str | None class QueryObjectFilterClause(TypedDict, total=False): col: Column op: str # pylint: disable=invalid-name val: FilterValues | None grain: str | None isExtra: bool | None class ExtraFiltersTimeColumnType(StrEnum): TIME_COL = "__time_col" TIME_GRAIN = "__time_grain" TIME_ORIGIN = "__time_origin" TIME_RANGE = "__time_range" class ExtraFiltersReasonType(StrEnum): NO_TEMPORAL_COLUMN = "no_temporal_column" COL_NOT_IN_DATASOURCE = "not_in_datasource" class FilterOperator(StrEnum): """ Operators used filter controls """ EQUALS = "==" NOT_EQUALS = "!=" GREATER_THAN = ">" LESS_THAN = "<" GREATER_THAN_OR_EQUALS = ">=" LESS_THAN_OR_EQUALS = "<=" LIKE = "LIKE" NOT_LIKE = "NOT LIKE" ILIKE = "ILIKE" NOT_ILIKE = "NOT ILIKE" IS_NULL = "IS NULL" IS_NOT_NULL = "IS NOT NULL" IN = "IN" NOT_IN = "NOT IN" IS_TRUE = "IS TRUE" IS_FALSE = "IS FALSE" TEMPORAL_RANGE = "TEMPORAL_RANGE" # Element-level operators for MULTI_VALUE (array) columns CONTAINS_ANY = "CONTAINS_ANY" CONTAINS_ALL = "CONTAINS_ALL" IS_EMPTY = "IS_EMPTY" IS_NOT_EMPTY = "IS_NOT_EMPTY" # Length (element-count) comparison operators for array columns LENGTH_EQUALS = "LENGTH_EQUALS" LENGTH_GREATER_THAN = "LENGTH_GREATER_THAN" LENGTH_LESS_THAN = "LENGTH_LESS_THAN" LENGTH_GREATER_THAN_OR_EQUALS = "LENGTH_GREATER_THAN_OR_EQUALS" LENGTH_LESS_THAN_OR_EQUALS = "LENGTH_LESS_THAN_OR_EQUALS" class FilterStringOperators(StrEnum): EQUALS = ("EQUALS",) NOT_EQUALS = ("NOT_EQUALS",) LESS_THAN = ("LESS_THAN",) GREATER_THAN = ("GREATER_THAN",) LESS_THAN_OR_EQUAL = ("LESS_THAN_OR_EQUAL",) GREATER_THAN_OR_EQUAL = ("GREATER_THAN_OR_EQUAL",) IN = ("IN",) NOT_IN = ("NOT_IN",) ILIKE = ("ILIKE",) LIKE = ("LIKE",) IS_NOT_NULL = ("IS_NOT_NULL",) IS_NULL = ("IS_NULL",) LATEST_PARTITION = ("LATEST_PARTITION",) IS_TRUE = ("IS_TRUE",) IS_FALSE = ("IS_FALSE",) CONTAINS_ANY = ("CONTAINS_ANY",) CONTAINS_ALL = ("CONTAINS_ALL",) IS_EMPTY = ("IS_EMPTY",) IS_NOT_EMPTY = ("IS_NOT_EMPTY",) LENGTH_EQUALS = ("LENGTH_EQUALS",) LENGTH_GREATER_THAN = ("LENGTH_GREATER_THAN",) LENGTH_LESS_THAN = ("LENGTH_LESS_THAN",) LENGTH_GREATER_THAN_OR_EQUALS = ("LENGTH_GREATER_THAN_OR_EQUALS",) LENGTH_LESS_THAN_OR_EQUALS = ("LENGTH_LESS_THAN_OR_EQUALS",) class PostProcessingBoxplotWhiskerType(StrEnum): """ Calculate cell contribution to row/column total """ TUKEY = "tukey" MINMAX = "min/max" PERCENTILE = "percentile" class PostProcessingContributionOrientation(StrEnum): """ Calculate cell contribution to row/column total """ ROW = "row" COLUMN = "column" class QuerySource(Enum): """ The source of a SQL query. """ CHART = 0 DASHBOARD = 1 SQL_LAB = 2 class QueryStatus(StrEnum): """Enum-type class for query statuses""" STOPPED = "stopped" FAILED = "failed" PENDING = "pending" RUNNING = "running" SCHEDULED = "scheduled" SUCCESS = "success" FETCHING = "fetching" TIMED_OUT = "timed_out" class DashboardStatus(StrEnum): """Dashboard status used for frontend filters""" PUBLISHED = "published" DRAFT = "draft" class ReservedUrlParameters(StrEnum): """ Reserved URL parameters that are used internally by Superset. These will not be passed to chart queries, as they control the behavior of the UI. """ STANDALONE = "standalone" EDIT_MODE = "edit" @staticmethod def is_standalone_mode() -> bool | None: standalone_param = request.args.get(ReservedUrlParameters.STANDALONE.value) standalone: bool | None = bool( standalone_param and standalone_param != "false" and standalone_param != "0" ) return standalone class RowLevelSecurityFilterType(StrEnum): REGULAR = "Regular" BASE = "Base" class ColumnTypeSource(Enum): GET_TABLE = 1 CURSOR_DESCRIPTION = 2 class ColumnSpec(NamedTuple): sqla_type: TypeEngine | str generic_type: GenericDataType is_dttm: bool python_date_format: str | None = None def parse_js_uri_path_item( item: str | None, unquote: bool = True, eval_undefined: bool = False ) -> str | None: """Parse an uri path item made with js. :param item: an uri path component :param unquote: Perform unquoting of string using urllib.parse.unquote_plus() :param eval_undefined: When set to True and item is either 'null' or 'undefined', assume item is undefined and return None. :return: Either None, the original item or unquoted item """ item = None if eval_undefined and item in ("null", "undefined") else item return unquote_plus(item) if unquote and item else item # Matches a safe, opaque token suitable for use as a cookie name. Restricting the # allowed characters prevents client-controlled input from injecting unexpected # cookie attributes or control characters. COOKIE_TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") def sanitize_cookie_token(token: str | None) -> str | None: """Return the token if it is a valid cookie name, otherwise None. The export endpoints echo a client-provided ``token`` query parameter back as a cookie name to signal download completion. Validate it against a strict allow-list before trusting it. :param token: the client-provided token value :return: the token if valid, else None """ if token and COOKIE_TOKEN_RE.match(token): return token return None def cast_to_num(value: float | int | str | None) -> float | int | None: """Casts a value to an int/float >>> cast_to_num('1 ') 1.0 >>> cast_to_num(' 2') 2.0 >>> cast_to_num('5') 5 >>> cast_to_num('5.2') 5.2 >>> cast_to_num(10) 10 >>> cast_to_num(10.1) 10.1 >>> cast_to_num(None) is None True >>> cast_to_num('this is not a string') is None True :param value: value to be converted to numeric representation :returns: value cast to `int` if value is all digits, `float` if `value` is decimal value and `None`` if it can't be converted """ if value is None: return None if isinstance(value, (int, float)): return value if value.isdigit(): return int(value) try: return float(value) except ValueError: return None def cast_to_boolean(value: Any) -> bool | None: """Casts a value to an int/float >>> cast_to_boolean(1) True >>> cast_to_boolean(0) False >>> cast_to_boolean(0.5) True >>> cast_to_boolean('true') True >>> cast_to_boolean('false') False >>> cast_to_boolean('False') False >>> cast_to_boolean(None) :param value: value to be converted to boolean representation :returns: value cast to `bool`. when value is 'true' or value that are not 0 converted into True. Return `None` if value is `None` """ if value is None: return None if isinstance(value, bool): return value if isinstance(value, (int, float)): return value != 0 if isinstance(value, str): return value.strip().lower() == "true" return False def error_msg_from_exception(ex: Exception) -> str: """Translate exception into error message Database have different ways to handle exception. This function attempts to make sense of the exception object and construct a human readable sentence. TODO(bkyryliuk): parse the Presto error message from the connection created via create_engine. engine = create_engine('presto://localhost:3506/silver') - gives an e.message as the str(dict) presto.connect('localhost', port=3506, catalog='silver') - as a dict. The latter version is parsed correctly by this function. """ msg = "" if hasattr(ex, "message"): if isinstance(ex.message, dict): msg = ex.message.get("message") # type: ignore elif ex.message: msg = ex.message return str(msg) or str(ex) def markdown(raw: str, markup_wrap: bool | None = False) -> str: """Render Markdown to sanitized HTML.""" safe_markdown_tags = { "h1", "h2", "h3", "h4", "h5", "h6", "b", "i", "strong", "em", "tt", "p", "br", "span", "div", "blockquote", "code", "hr", "ul", "ol", "li", "dd", "dt", "img", "a", } safe_markdown_attrs = { "img": {"src", "alt", "title"}, "a": {"href", "alt", "title", "target"}, } safe = md.markdown( raw or "", extensions=[ "markdown.extensions.tables", "markdown.extensions.fenced_code", "markdown.extensions.codehilite", ], ) # pylint: disable=no-member # nh3 preserves supported link attributes and enforces a safe rel value. safe = nh3.clean(safe, tags=safe_markdown_tags, attributes=safe_markdown_attrs) if markup_wrap: safe = Markup(safe) return safe def sanitize_svg_content(svg_content: str) -> str: """Basic SVG protection - remove obvious XSS vectors, trust admin input otherwise. Minimal protection approach that removes scripts and javascript: URLs while preserving all legitimate SVG features. Assumes admin-provided content. Args: svg_content: Raw SVG content string Returns: str: SVG content with obvious XSS vectors removed """ if not svg_content or not svg_content.strip(): return "" # Minimal protection: remove obvious malicious content, preserve all SVG features # The closing tag pattern tolerates attributes/whitespace after "script" # (e.g. ""), which browsers still parse as a valid closer. content = re.sub( r"]*>.*?]*>", "", svg_content, flags=re.IGNORECASE | re.DOTALL, ) # Second pass: an unterminated fragment too. content = re.sub(r"]*>?", "", content, flags=re.IGNORECASE) content = re.sub(r"javascript:", "", content, flags=re.IGNORECASE) content = re.sub(r"data:[^;]*;[^,]*,.*javascript", "", content, flags=re.IGNORECASE) # Remove event handlers (simple catch-all approach) content = re.sub(r"\bon\w+\s*=", "", content, flags=re.IGNORECASE) # Remove other suspicious patterns content = re.sub( r"]*>.*?", "", content, flags=re.IGNORECASE | re.DOTALL ) content = re.sub( r"]*>.*?", "", content, flags=re.IGNORECASE | re.DOTALL ) content = re.sub(r"]*>", "", content, flags=re.IGNORECASE) return content def sanitize_url(url: str) -> str: """Sanitize URL using urllib.parse to block dangerous schemes. Simple validation using standard library. Allows relative URLs and safe absolute URLs while blocking javascript: and other dangerous schemes. Args: url: Raw URL string Returns: str: Sanitized URL or empty string if dangerous """ if not url or not url.strip(): return "" url = url.strip() # Relative URLs are safe if url.startswith("/"): return url try: from urllib.parse import urlparse parsed = urlparse(url) # Allow safe schemes only if parsed.scheme.lower() in {"http", "https", ""}: return url # Block everything else (javascript:, data:, etc.) return "" except Exception: return "" def readfile(file_path: str) -> str | None: with open(file_path) as f: content = f.read() return content def generic_find_constraint_name( table: str, columns: set[str], referenced: str, database: SQLAlchemy ) -> str | None: """Utility to find a constraint name in alembic migrations""" tbl = sa.Table(table, database.metadata, autoload_with=database.engine) for fk in tbl.foreign_key_constraints: if fk.referred_table.name == referenced and set(fk.column_keys) == columns: return fk.name return None def generic_find_fk_constraint_name( table: str, columns: set[str], referenced: str, insp: Inspector ) -> str | None: """Utility to find a foreign-key constraint name in alembic migrations""" for fk in insp.get_foreign_keys(table): if ( fk["referred_table"] == referenced and set(fk["referred_columns"]) == columns ): return fk["name"] return None def generic_find_fk_constraint_names( # pylint: disable=invalid-name table: str, columns: set[str], referenced: str, insp: Inspector ) -> set[str]: """Utility to find foreign-key constraint names in alembic migrations""" names = set() for fk in insp.get_foreign_keys(table): if ( fk["referred_table"] == referenced and set(fk["referred_columns"]) == columns ): names.add(fk["name"]) return names def generic_find_uq_constraint_name( table: str, columns: Collection[str], insp: Inspector ) -> str | None: """Utility to find a unique constraint name in alembic migrations. ``columns`` is coerced to a set before comparison. Historically the parameter was annotated ``set[str]`` but compared with ``==`` against a set — a caller passing a list (as migration ``df3d7e2eb9a4`` did) silently never matched, because ``list == set`` is always ``False``. Coercing removes that foot-gun for future callers. """ target = set(columns) for uq in insp.get_unique_constraints(table): if target == set(uq["column_names"]): return uq["name"] return None def get_datasource_full_name( database_name: str, datasource_name: str, catalog: str | None = None, schema: str | None = None, ) -> str: parts = [database_name, catalog, schema, datasource_name] return ".".join([f"[{part}]" for part in parts if part]) class SigalrmTimeout: """ To be used in a ``with`` block and timeout its content. """ def __init__(self, seconds: int = 1, error_message: str = "Timeout") -> None: self.seconds = seconds self.error_message = error_message def handle_timeout( # pylint: disable=unused-argument self, signum: int, frame: Any ) -> None: logger.error("Process timed out", exc_info=True) raise SupersetTimeoutException( error_type=SupersetErrorType.BACKEND_TIMEOUT_ERROR, message=self.error_message, level=ErrorLevel.ERROR, extra={"timeout": self.seconds}, ) def __enter__(self) -> None: try: if threading.current_thread() == threading.main_thread(): signal.signal(signal.SIGALRM, self.handle_timeout) signal.alarm(self.seconds) except ValueError as ex: logger.warning("timeout can't be used in the current context") logger.exception(ex) def __exit__( # pylint: disable=redefined-outer-name,redefined-builtin self, type: Any, value: Any, traceback: TracebackType ) -> None: try: signal.alarm(0) except ValueError as ex: logger.warning("timeout can't be used in the current context") logger.exception(ex) class TimerTimeout: def __init__(self, seconds: int = 1, error_message: str = "Timeout") -> None: self.seconds = seconds self.error_message = error_message self.timer = threading.Timer(seconds, _thread.interrupt_main) def __enter__(self) -> None: self.timer.start() def __exit__( # pylint: disable=redefined-outer-name,redefined-builtin self, type: Any, value: Any, traceback: TracebackType ) -> None: self.timer.cancel() if type is KeyboardInterrupt: # raised by _thread.interrupt_main raise SupersetTimeoutException( error_type=SupersetErrorType.BACKEND_TIMEOUT_ERROR, message=self.error_message, level=ErrorLevel.ERROR, extra={"timeout": self.seconds}, ) # Windows has no support for SIGALRM, so we use the timer based timeout timeout: type[TimerTimeout] | type[SigalrmTimeout] = ( TimerTimeout if platform.system() == "Windows" else SigalrmTimeout ) def pessimistic_connection_handling(some_engine: Engine) -> None: @event.listens_for(some_engine, "engine_connect") def ping_connection(connection: Connection, branch: bool) -> None: if branch: # 'branch' refers to a sub-connection of a connection, # we don't want to bother pinging on these. return # turn off 'close with result'. This flag is only used with # 'connectionless' execution, otherwise will be False in any case save_should_close_with_result = connection.should_close_with_result connection.should_close_with_result = False try: # run a SELECT 1. use a core select() so that # the SELECT of a scalar value without a table is # appropriately formatted for the backend connection.scalar(select(1)) connection.rollback() # pylint: disable=consider-using-transaction except exc.DBAPIError as err: # catch SQLAlchemy's DBAPIError, which is a wrapper # for the DBAPI's exception. It includes a .connection_invalidated # attribute which specifies if this connection is a 'disconnect' # condition, which is based on inspection of the original exception # by the dialect in use. if err.connection_invalidated: # run the same SELECT again - the connection will re-validate # itself and establish a new connection. The disconnect detection # here also causes the whole connection pool to be invalidated # so that all stale connections are discarded. connection.scalar(select(1)) connection.rollback() # pylint: disable=consider-using-transaction else: raise finally: # restore 'close with result' connection.should_close_with_result = save_should_close_with_result if some_engine.dialect.name == "sqlite": @event.listens_for(some_engine, "connect") def set_sqlite_pragma( # pylint: disable=unused-argument connection: sqlite3.Connection, *args: Any, ) -> None: r""" Enable foreign key support for SQLite. :param connection: The SQLite connection :param \*args: Additional positional arguments :see: https://docs.sqlalchemy.org/en/latest/dialects/sqlite.html """ with closing(connection.cursor()) as cursor: cursor.execute("PRAGMA foreign_keys=ON") def send_email_smtp( # pylint: disable=invalid-name,too-many-arguments,too-many-locals to: str, subject: str, html_content: str, config: dict[str, Any], files: list[str] | None = None, data: dict[str, bytes | str] | None = None, pdf: dict[str, bytes] | None = None, images: dict[str, bytes] | None = None, dryrun: bool = False, cc: str | None = None, bcc: str | None = None, mime_subtype: str = "mixed", header_data: HeaderDataType | None = None, ) -> None: """ Send an email with html content, eg: send_email_smtp( 'test@example.com', 'foo', 'Foo bar',['/dev/null'], dryrun=True) """ smtp_mail_from = config["SMTP_MAIL_FROM"] smtp_mail_to = recipients_string_to_list(to) msg = MIMEMultipart(mime_subtype) # Strip CR/LF from the subject so the value cannot inject additional # email headers via header folding/splitting. subject = subject.replace("\r", "").replace("\n", " ").strip() msg["Subject"] = subject msg["From"] = smtp_mail_from msg["To"] = ", ".join(smtp_mail_to) msg.preamble = "This is a multi-part message in MIME format." recipients = smtp_mail_to if cc: smtp_mail_cc = recipients_string_to_list(cc) msg["Cc"] = ", ".join(smtp_mail_cc) recipients = recipients + smtp_mail_cc smtp_mail_bcc = [] if bcc: # don't add bcc in header smtp_mail_bcc = recipients_string_to_list(bcc) recipients = recipients + smtp_mail_bcc msg["Date"] = formatdate(localtime=True) mime_text = MIMEText(html_content, "html") msg.attach(mime_text) # Attach files by reading them from disk for fname in files or []: basename = os.path.basename(fname) with open(fname, "rb") as f: msg.attach(build_email_attachment(basename, f.read())) # Attach any files passed directly for name, body in (data or {}).items(): msg.attach(build_email_attachment(name, body)) for name, body_pdf in (pdf or {}).items(): msg.attach(build_email_attachment(name, body_pdf)) # Attach any inline images, which may be required for display in # HTML content (inline) for msgid, imgdata in (images or {}).items(): formatted_time = formatdate(localtime=True) file_name = f"{subject} {formatted_time}" image = MIMEImage(imgdata, name=file_name) image.add_header("Content-ID", f"<{msgid}>") image.add_header("Content-Disposition", "inline") msg.attach(image) msg_mutator = config["EMAIL_HEADER_MUTATOR"] # the base notification returns the message without any editing. new_msg = msg_mutator(msg, **(header_data or {})) new_to = new_msg["To"].split(", ") if "To" in new_msg else [] new_cc = new_msg["Cc"].split(", ") if "Cc" in new_msg else [] new_recipients = new_to + new_cc + smtp_mail_bcc if set(new_recipients) != set(recipients): recipients = new_recipients send_mime_email(smtp_mail_from, recipients, new_msg, config, dryrun=dryrun) def send_mime_email( e_from: str, e_to: list[str], mime_msg: MIMEMultipart, config: dict[str, Any], dryrun: bool = False, ) -> None: smtp_host = config["SMTP_HOST"] smtp_port = config["SMTP_PORT"] smtp_user = config["SMTP_USER"] smtp_password = config["SMTP_PASSWORD"] smtp_starttls = config["SMTP_STARTTLS"] smtp_ssl = config["SMTP_SSL"] smtp_ssl_server_auth = config["SMTP_SSL_SERVER_AUTH"] # A missing timeout means the socket blocks forever when the SMTP server is # unreachable, wedging the report schedule in the WORKING state. Fall back to # the key being absent for backwards compatibility with custom configs. # Keep this fallback in sync with the SMTP_TIMEOUT default in config.py. smtp_timeout = config.get("SMTP_TIMEOUT", 30) if dryrun: logger.info("Dryrun enabled, email notification content is below:") logger.info(mime_msg.as_string()) return # Default ssl context is SERVER_AUTH using the default system # root CA certificates ssl_context = ssl.create_default_context() if smtp_ssl_server_auth else None smtp = ( smtplib.SMTP_SSL( smtp_host, smtp_port, context=ssl_context, timeout=smtp_timeout ) if smtp_ssl else smtplib.SMTP(smtp_host, smtp_port, timeout=smtp_timeout) ) try: if smtp_starttls: smtp.starttls(context=ssl_context) if smtp_user and smtp_password: smtp.login(smtp_user, smtp_password) logger.debug("Sent an email to %s", str(e_to)) smtp.sendmail(e_from, e_to, mime_msg.as_string()) finally: # Always release the socket; the new timeout means starttls/login/ # sendmail can raise, and a skipped quit() would leak connections in # the long-lived worker process. try: smtp.quit() except smtplib.SMTPException: pass def recipients_string_to_list(address_string: str | None) -> list[str]: """ Returns the list of target recipients for alerts and reports. Strips values and converts a comma/semicolon separated string into a list. """ address_string_list: list[str] = [] if isinstance(address_string, str): address_string_list = re.split(r",|\s|;", address_string) return [x.strip() for x in address_string_list if x.strip()] def choicify(values: Iterable[Any]) -> list[tuple[Any, Any]]: """Takes an iterable and makes an iterable of tuples with it""" return [(v, v) for v in values] def zlib_compress(data: bytes | str) -> bytes: """ Compress things in a py2/3 safe fashion >>> json_str = '{"test": 1}' >>> blob = zlib_compress(json_str) """ if isinstance(data, str): return zlib.compress(bytes(data, "utf-8")) return zlib.compress(data) def zlib_decompress(blob: bytes, decode: bool | None = True) -> bytes | str: """ Decompress things to a string in a py2/3 safe fashion >>> json_str = '{"test": 1}' >>> blob = zlib_compress(json_str) >>> got_str = zlib_decompress(blob) >>> got_str == json_str True """ if isinstance(blob, bytes): decompressed = zlib.decompress(blob) else: decompressed = zlib.decompress(bytes(blob, "utf-8")) return decompressed.decode("utf-8") if decode else decompressed def simple_filter_to_adhoc( filter_clause: QueryObjectFilterClause, clause: str = "where", ) -> AdhocFilterClause: result: AdhocFilterClause = { "clause": clause.upper(), "expressionType": "SIMPLE", "comparator": filter_clause.get("val"), "operator": filter_clause["op"], "subject": cast(str, filter_clause["col"]), } if filter_clause.get("isExtra"): result["isExtra"] = True result["filterOptionName"] = hash_from_dict(cast(dict[Any, Any], result)) return result def form_data_to_adhoc(form_data: dict[str, Any], clause: str) -> AdhocFilterClause: if clause not in ("where", "having"): raise ValueError(__("Unsupported clause type: %(clause)s", clause=clause)) result: AdhocFilterClause = { "clause": clause.upper(), "expressionType": "SQL", "sqlExpression": form_data.get(clause), } result["filterOptionName"] = hash_from_dict(cast(dict[Any, Any], result)) return result def _update_existing_temporal_filter( temporal_filter: AdhocFilterClause, granularity_sqla_override: str | None, time_range: str | None, chart_has_granularity_sqla: bool, ) -> None: """Update an existing temporal filter with new subject/comparator.""" if ( granularity_sqla_override is not None and temporal_filter.get("expressionType") == "SIMPLE" ): temporal_filter["subject"] = granularity_sqla_override if time_range and not chart_has_granularity_sqla: temporal_filter["comparator"] = time_range def _create_temporal_filter( granularity_sqla: str, time_range: str, ) -> AdhocFilterClause: """Create a new TEMPORAL_RANGE adhoc filter.""" new_filter: AdhocFilterClause = { "clause": "WHERE", "expressionType": "SIMPLE", "operator": FilterOperator.TEMPORAL_RANGE, "subject": granularity_sqla, "comparator": time_range, "isExtra": True, } new_filter["filterOptionName"] = hash_from_dict(cast(dict[Any, Any], new_filter)) return new_filter def merge_extra_form_data(form_data: dict[str, Any]) -> None: # noqa: C901 """ Merge extra form data (appends and overrides) into the main payload and add applied time extras to the payload. """ filter_keys = ["filters", "adhoc_filters"] extra_form_data = form_data.pop("extra_form_data", {}) append_filters: list[QueryObjectFilterClause] = extra_form_data.get("filters", None) # merge append extras for key in [key for key in EXTRA_FORM_DATA_APPEND_KEYS if key not in filter_keys]: extra_value = getattr(extra_form_data, key, {}) form_value = getattr(form_data, key, {}) form_value.update(extra_value) if form_value: form_data["key"] = extra_value # map regular extras that apply to form data properties for src_key, target_key in EXTRA_FORM_DATA_OVERRIDE_REGULAR_MAPPINGS.items(): value = extra_form_data.get(src_key) if value is not None: form_data[target_key] = value # map extras that apply to form data extra properties extras = form_data.get("extras", {}) for key in EXTRA_FORM_DATA_OVERRIDE_EXTRA_KEYS: value = extra_form_data.get(key) if value is not None: extras[key] = value if extras: form_data["extras"] = extras adhoc_filters: list[AdhocFilterClause] = form_data.get("adhoc_filters", []) form_data["adhoc_filters"] = adhoc_filters append_adhoc_filters: list[AdhocFilterClause] = extra_form_data.get( "adhoc_filters", [] ) adhoc_filters.extend( {"isExtra": True, **adhoc_filter} for adhoc_filter in append_adhoc_filters ) if append_filters: for key, value in form_data.items(): if re.match("adhoc_filter.*", key): value.extend( simple_filter_to_adhoc({"isExtra": True, **fltr}) for fltr in append_filters if fltr ) granularity_sqla_override = extra_form_data.get("granularity_sqla") time_range = form_data.get("time_range") chart_has_granularity_sqla = bool(form_data.get("granularity_sqla")) temporal_filters = [ adhoc_filter for adhoc_filter in adhoc_filters if adhoc_filter.get("operator") == FilterOperator.TEMPORAL_RANGE ] for temporal_filter in temporal_filters: _update_existing_temporal_filter( temporal_filter, granularity_sqla_override, time_range, chart_has_granularity_sqla, ) if ( not temporal_filters and granularity_sqla_override is not None and time_range is not None ): new_temporal_filter = _create_temporal_filter( granularity_sqla_override, cast(str, time_range), ) adhoc_filters.append(new_temporal_filter) def merge_extra_filters(form_data: dict[str, Any]) -> None: # noqa: C901 # extra_filters are temporary/contextual filters (using the legacy constructs) # that are external to the slice definition. We use those for dynamic # interactive filters. # Note extra_filters only support simple filters. form_data.setdefault("applied_time_extras", {}) adhoc_filters = form_data.get("adhoc_filters", []) form_data["adhoc_filters"] = adhoc_filters merge_extra_form_data(form_data) if "extra_filters" in form_data: # __form and __to are special extra_filters that target time # boundaries. The rest of extra_filters are simple # [column_name in list_of_values]. `__` prefix is there to avoid # potential conflicts with column that would be named `from` or `to` date_options = { "__time_range": "time_range", "__time_col": "granularity_sqla", "__time_grain": "time_grain_sqla", } # Grab list of existing filters 'keyed' on the column and operator def get_filter_key(f: dict[str, Any]) -> str: if "expressionType" in f: return f"{f['subject']}__{f['operator']}" return f"{f['col']}__{f['op']}" existing_filters = {} for existing in adhoc_filters: if ( existing["expressionType"] == "SIMPLE" and existing.get("comparator") is not None and existing.get("subject") is not None ): existing_filters[get_filter_key(existing)] = existing["comparator"] for filtr in form_data[ # pylint: disable=too-many-nested-blocks "extra_filters" ]: filtr["isExtra"] = True # Pull out time filters/options and merge into form data filter_column = filtr["col"] if time_extra := date_options.get(filter_column): time_extra_value = filtr.get("val") if time_extra_value and time_extra_value != NO_TIME_RANGE: form_data[time_extra] = time_extra_value form_data["applied_time_extras"][filter_column] = time_extra_value elif filtr["val"]: # Merge column filters if (filter_key := get_filter_key(filtr)) in existing_filters: # Check if the filter already exists if isinstance(filtr["val"], list): if isinstance(existing_filters[filter_key], list): # Add filters for unequal lists # order doesn't matter if set(existing_filters[filter_key]) != set(filtr["val"]): adhoc_filters.append(simple_filter_to_adhoc(filtr)) else: adhoc_filters.append(simple_filter_to_adhoc(filtr)) else: # Do not add filter if same value already exists if filtr["val"] != existing_filters[filter_key]: adhoc_filters.append(simple_filter_to_adhoc(filtr)) else: # Filter not found, add it adhoc_filters.append(simple_filter_to_adhoc(filtr)) # Remove extra filters from the form data since no longer needed del form_data["extra_filters"] def merge_request_params(form_data: dict[str, Any], params: dict[str, Any]) -> None: """ Merge request parameters to the key `url_params` in form_data. Only updates or appends parameters to `form_data` that are defined in `params; preexisting parameters not defined in params are left unchanged. :param form_data: object to be updated :param params: request parameters received via query string """ url_params = form_data.get("url_params", {}) for key, value in params.items(): if key in ("form_data", "r"): continue url_params[key] = value form_data["url_params"] = url_params def user_label(user: User) -> str | None: """Given a user ORM FAB object, returns a label""" if user: if user.first_name and user.last_name: return user.first_name + " " + user.last_name return user.username return None def get_example_default_schema() -> str | None: """ Return the default schema of the examples database, if any. """ database = get_example_database() with database.get_sqla_engine() as engine: return inspect(engine).default_schema_name def backend() -> str: return get_example_database().backend def is_adhoc_metric(metric: Metric) -> TypeGuard[AdhocMetric]: return isinstance(metric, dict) and "expressionType" in metric def is_adhoc_column(column: Column) -> TypeGuard[AdhocColumn]: return isinstance(column, dict) and ({"label", "sqlExpression"}).issubset( column.keys() ) def is_base_axis(column: Column) -> bool: return is_adhoc_column(column) and column.get("columnType") == "BASE_AXIS" def get_base_axis_columns(columns: list[Column] | None) -> list[Column]: return [column for column in columns or [] if is_base_axis(column)] def get_non_base_axis_columns(columns: list[Column] | None) -> list[Column]: return [column for column in columns or [] if not is_base_axis(column)] def get_base_axis_labels(columns: list[Column] | None) -> tuple[str, ...]: return tuple(get_column_name(column) for column in get_base_axis_columns(columns)) def get_x_axis_label(columns: list[Column] | None) -> str | None: labels = get_base_axis_labels(columns) return labels[0] if labels else None def get_column_name(column: Column, verbose_map: dict[str, Any] | None = None) -> str: """ Extract label from column :param column: object to extract label from :param verbose_map: verbose_map from dataset for optional mapping from raw name to verbose name :return: String representation of column :raises ValueError: if metric object is invalid """ if hasattr(column, "column_name"): column_name = getattr(column, "column_name", "") verbose_name = getattr(column, "verbose_name", "") return verbose_name or column_name if isinstance(column, dict): if label := column.get("label"): return label if expr := column.get("sqlExpression"): return expr if isinstance(column, str): verbose_map = verbose_map or {} return verbose_map.get(column, column) raise ValueError("Missing label") def get_metric_name(metric: Metric, verbose_map: dict[str, Any] | None = None) -> str: """ Extract label from metric :param metric: object to extract label from :param verbose_map: verbose_map from dataset for optional mapping from raw name to verbose name :return: String representation of metric :raises ValueError: if metric object is invalid """ if is_adhoc_metric(metric): if label := metric.get("label"): return label if (expression_type := metric.get("expressionType")) == "SQL": if sql_expression := metric.get("sqlExpression"): return sql_expression if expression_type == "SIMPLE": column: AdhocMetricColumn = metric.get("column") or {} column_name = column.get("column_name") aggregate = metric.get("aggregate") if column and aggregate: return f"{aggregate}({column_name})" if column_name: return column_name if isinstance(metric, str): verbose_map = verbose_map or {} return verbose_map.get(metric, metric) raise ValueError(__("Invalid metric object: %(metric)s", metric=str(metric))) def get_column_names( columns: Sequence[Column] | None, verbose_map: dict[str, Any] | None = None, ) -> list[str]: return [ column for column in [get_column_name(column, verbose_map) for column in columns or []] if column ] def get_metric_names( metrics: Sequence[Metric] | None, verbose_map: dict[str, Any] | None = None, ) -> list[str]: return [ metric for metric in [get_metric_name(metric, verbose_map) for metric in metrics or []] if metric ] def get_first_metric_name( metrics: Sequence[Metric] | None, verbose_map: dict[str, Any] | None = None, ) -> str | None: metric_labels = get_metric_names(metrics, verbose_map) return metric_labels[0] if metric_labels else None def ensure_path_exists(path: str) -> None: try: os.makedirs(path) except OSError as ex: if not (os.path.isdir(path) and ex.errno == errno.EEXIST): raise def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name form_data: FormData, ) -> None: if not form_data.get("adhoc_filters"): adhoc_filters: list[AdhocFilterClause] = [] form_data["adhoc_filters"] = adhoc_filters for clause in ("having", "where"): if clause in form_data and form_data[clause] != "": adhoc_filters.append(form_data_to_adhoc(form_data, clause)) if "filters" in form_data: adhoc_filters.extend( simple_filter_to_adhoc(fltr, "where") for fltr in form_data["filters"] if fltr is not None ) for key in ("filters", "having", "where"): if key in form_data: del form_data[key] def split_adhoc_filters_into_base_filters( # pylint: disable=invalid-name form_data: FormData, engine: str | None = None, # pylint: disable=unused-argument ) -> None: """ Mutates form data to restructure the adhoc filters in the form of the three base filters, `where`, `having`, and `filters` which represent free form where sql, free form having sql, and structured where clauses. ``engine`` is retained for backwards compatibility and is unused: clauses are validated downstream, after Jinja templates are rendered. """ adhoc_filters = form_data.get("adhoc_filters") if isinstance(adhoc_filters, list): simple_where_filters = [] sql_where_filters = [] sql_having_filters = [] for adhoc_filter in adhoc_filters: expression_type = adhoc_filter.get("expressionType") clause = adhoc_filter.get("clause") if expression_type == "SIMPLE": if clause == "WHERE": simple_where_filters.append( { "col": adhoc_filter.get("subject"), "op": adhoc_filter.get("operator"), "val": adhoc_filter.get("comparator"), } ) elif expression_type == "SQL": sql_expression = adhoc_filter.get("sqlExpression") # keep a trailing line comment from swallowing the " AND " join if sql_expression and "--" in sql_expression: sql_expression = f"{sql_expression}\n" if clause == "WHERE": sql_where_filters.append(sql_expression) elif clause == "HAVING": sql_having_filters.append(sql_expression) form_data["where"] = " AND ".join([f"({sql})" for sql in sql_where_filters]) form_data["having"] = " AND ".join([f"({sql})" for sql in sql_having_filters]) form_data["filters"] = simple_where_filters def get_user() -> User | None: """ Get the current user (if defined). :returns: The current user """ return g.user if hasattr(g, "user") else None def get_username() -> str | None: """ Get username (if defined) associated with the current user. :returns: The username """ try: return g.user.username except Exception: # pylint: disable=broad-except return None def get_user_id() -> int | None: """ Get the user identifier (if defined) associated with the current user. Though the Flask-AppBuilder `User` and Flask-Login `AnonymousUserMixin` and `UserMixin` models provide a convenience `get_id` method, for generality, the identifier is encoded as a `str` whereas in Superset all identifiers are encoded as an `int`. returns: The user identifier """ try: return g.user.id except Exception: # pylint: disable=broad-except return None def get_user_email() -> str | None: """ Get the email (if defined) associated with the current user. :returns: The email """ try: return g.user.email except Exception: # pylint: disable=broad-except return None @contextmanager def override_user(user: User | None, force: bool = True) -> Iterator[Any]: """ Temporarily override the current user per `flask.g` with the specified user. Sometimes, often in the context of async Celery tasks, it is useful to switch the current user (which may be undefined) to different one, execute some SQLAlchemy tasks et al. and then revert back to the original one. :param user: The override user :param force: Whether to override the current user if set """ if hasattr(g, "user"): if force or g.user is None: current = g.user g.user = user yield g.user = current else: yield else: g.user = user yield delattr(g, "user") def parse_ssl_cert(certificate: str) -> Certificate: """ Parses the contents of a certificate and returns a valid certificate object if valid. :param certificate: Contents of certificate file :return: Valid certificate instance :raises CertificateException: If certificate is not valid/unparseable """ try: return load_pem_x509_certificate(certificate.encode("utf-8"), default_backend()) except ValueError as ex: raise CertificateException("Invalid certificate") from ex def create_ssl_cert_file(certificate: str) -> str: """ This creates a certificate file that can be used to validate HTTPS sessions. A certificate is only written to disk once; on subsequent calls, only the path of the existing certificate is returned. :param certificate: The contents of the certificate :return: The path to the certificate file :raises CertificateException: If certificate is not valid/unparseable """ filename = f"{hash_from_str(certificate)}.crt" # pylint: disable=import-outside-toplevel cert_dir = app.config["SSL_CERT_PATH"] path = cert_dir if cert_dir else tempfile.gettempdir() path = os.path.join(path, filename) if not os.path.exists(path): # Validate certificate prior to persisting to temporary directory parse_ssl_cert(certificate) with open(path, "w") as cert_file: cert_file.write(certificate) return path def time_function( func: Callable[..., FlaskResponse], *args: Any, **kwargs: Any ) -> tuple[float, Any]: """ Measures the amount of time a function takes to execute in ms :param func: The function execution time to measure :param args: args to be passed to the function :param kwargs: kwargs to be passed to the function :return: A tuple with the duration and response from the function """ start = default_timer() response = func(*args, **kwargs) stop = default_timer() return (stop - start) * 1000.0, response def MediumText() -> Variant: # pylint:disable=invalid-name # noqa: N802 return Text().with_variant(MEDIUMTEXT(), "mysql") def LongText() -> Variant: # pylint:disable=invalid-name # noqa: N802 return Text().with_variant(LONGTEXT(), "mysql") def shortid() -> str: return f"{uuid.uuid4()}"[-12:] class DatasourceName(NamedTuple): table: str schema: str catalog: str | None = None def get_stacktrace() -> str | None: # pylint: disable=import-outside-toplevel if app.config["SHOW_STACKTRACE"]: return traceback.format_exc() return None def split( string: str, delimiter: str = " ", quote: str = '"', escaped_quote: str = r"\"" ) -> Iterator[str]: """ A split function that is aware of quotes and parentheses. :param string: string to split :param delimiter: string defining where to split, usually a comma or space :param quote: string, either a single or a double quote :param escaped_quote: string representing an escaped quote :return: list of strings """ parens = 0 quotes = False i = 0 for j, character in enumerate(string): complete = parens == 0 and not quotes if complete and character == delimiter: yield string[i:j] i = j + len(delimiter) elif character == "(": parens += 1 elif character == ")": parens -= 1 elif character == quote: if quotes and string[j - len(escaped_quote) + 1 : j + 1] != escaped_quote: quotes = False elif not quotes: quotes = True yield string[i:] T = TypeVar("T") def as_list(x: T | list[T]) -> list[T]: """ Wrap an object in a list if it's not a list. :param x: The object :returns: A list wrapping the object if it's not already a list """ return x if isinstance(x, list) else [x] def get_form_data_token(form_data: dict[str, Any]) -> str: """ Return the token contained within form data or generate a new one. :param form_data: chart form data :return: original token if predefined, otherwise new uuid4 based token """ return form_data.get("token") or "token_" + uuid.uuid4().hex[:8] def get_column_name_from_column(column: Column) -> str | None: """ Extract the physical column that a column is referencing. If the column is an adhoc column, always returns `None`. :param column: Physical and ad-hoc column :return: column name if physical column, otherwise None """ if is_adhoc_column(column): return None return column # type: ignore def get_column_names_from_columns(columns: list[Column]) -> list[str]: """ Extract the physical columns that a list of columns are referencing. Ignore adhoc columns :param columns: Physical and adhoc columns :return: column names of all physical columns """ return [col for col in map(get_column_name_from_column, columns) if col] def get_column_name_from_metric(metric: Metric) -> str | None: """ Extract the column that a metric is referencing. If the metric isn't a simple metric, always returns `None`. :param metric: Ad-hoc metric :return: column name if simple metric, otherwise None """ if is_adhoc_metric(metric): metric = cast(AdhocMetric, metric) if metric["expressionType"] == AdhocMetricExpressionType.SIMPLE: column = metric["column"] if column: return column["column_name"] return None def get_column_names_from_metrics(metrics: list[Metric]) -> list[str]: """ Extract the columns that a list of metrics are referencing. Excludes all SQL metrics. :param metrics: Ad-hoc metric :return: column name if simple metric, otherwise None """ return [col for col in map(get_column_name_from_metric, metrics) if col] def map_sql_type_to_inferred_type(sql_type: Optional[str]) -> str: """ Map a SQL type to a type string recognized by pandas' `infer_objects` method. If the SQL type is not recognized, the function will return "string" as the default type. :param sql_type: SQL type to map :return: string type recognized by pandas """ if not sql_type: return "string" # If no SQL type is provided, return "string" as default # Use regular expressions to check the SQL type. The first match is returned. for pattern, inferred_type in TYPE_MAPPING.items(): if pattern.search(sql_type): return inferred_type return "string" # If no match is found, return "string" as default def get_metric_type_from_column(column: Any, datasource: Explorable) -> str: """ Determine the metric type from a given column in a datasource. This function checks if the specified column is a metric in the provided datasource. If it is, it extracts the SQL expression associated with the metric and attempts to identify the aggregation operation used within the expression (e.g., SUM, COUNT, etc.). It then maps the operation to a corresponding GenericDataType. :param column: The column name or identifier to check. :param datasource: The datasource containing metrics to search within. :return: The inferred metric type as a string, or an empty string if the column is not a metric or no valid operation is found. """ metric = next( (m for m in datasource.metrics if m.metric_name == column), None, ) if metric is None: return "" expression: str = metric.expression match = re.match( r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX|FIRST|LAST" r"|MEDIAN|STDDEV_SAMP|VAR_SAMP)\s*\((.*)\)", expression, re.IGNORECASE, ) if match: operation = match.group(1).upper() return METRIC_MAP_TYPE.get(operation, "") logger.debug("Unexpected metric expression type: %s", expression) return "" def extract_dataframe_dtypes( df: pd.DataFrame, datasource: Explorable | None = None, ) -> list[GenericDataType]: """Serialize pandas/numpy dtypes to generic types""" # omitting string types as those will be the default type inferred_type_map: dict[str, GenericDataType] = { "floating": GenericDataType.NUMERIC, "integer": GenericDataType.NUMERIC, "mixed-integer-float": GenericDataType.NUMERIC, "decimal": GenericDataType.NUMERIC, "boolean": GenericDataType.BOOLEAN, "datetime64": GenericDataType.TEMPORAL, "datetime": GenericDataType.TEMPORAL, "date": GenericDataType.TEMPORAL, } columns_by_name: dict[str, Any] = {} if datasource: for column in datasource.columns: if isinstance(column, dict): if column_name := column.get("column_name"): columns_by_name[column_name] = column else: columns_by_name[column.column_name] = column generic_types: list[GenericDataType] = [] for i, column in enumerate(df.columns): column_object = columns_by_name.get(str(column)) series = df.iloc[:, i] inferred_type: str = "" if series.isna().all(): sql_type: Optional[str] = "" if datasource and hasattr(datasource, "columns_types"): if column in datasource.columns_types: sql_type = datasource.columns_types.get(column) inferred_type = map_sql_type_to_inferred_type(sql_type) else: inferred_type = get_metric_type_from_column(column, datasource) else: inferred_type = infer_dtype(series) if isinstance(column_object, dict): generic_type = ( GenericDataType.TEMPORAL if column_object and column_object.get("is_dttm") else inferred_type_map.get(inferred_type, GenericDataType.STRING) ) else: generic_type = ( GenericDataType.TEMPORAL if column_object and column_object.is_dttm else inferred_type_map.get(inferred_type, GenericDataType.STRING) ) generic_types.append(generic_type) return generic_types def extract_column_dtype(col: ColumnMetadata) -> GenericDataType: # Check for temporal type if hasattr(col, "is_temporal") and col.is_temporal: return GenericDataType.TEMPORAL if col.is_dttm: return GenericDataType.TEMPORAL # Check for numeric type if hasattr(col, "is_numeric") and col.is_numeric: return GenericDataType.NUMERIC # TODO: add check for boolean data type when proper support is added return GenericDataType.STRING def is_test() -> bool: return parse_boolean_string(os.environ.get("SUPERSET_TESTENV", "false")) def get_time_filter_status( datasource: Explorable, applied_time_extras: dict[str, str], ) -> tuple[list[dict[str, str]], list[dict[str, str]]]: temporal_columns: set[Any] = { col.column_name for col in datasource.columns if col.is_dttm } applied: list[dict[str, str]] = [] rejected: list[dict[str, str]] = [] if time_column := applied_time_extras.get(ExtraFiltersTimeColumnType.TIME_COL): if time_column in temporal_columns: applied.append({"column": ExtraFiltersTimeColumnType.TIME_COL}) else: rejected.append( { "reason": ExtraFiltersReasonType.COL_NOT_IN_DATASOURCE, "column": ExtraFiltersTimeColumnType.TIME_COL, } ) if ExtraFiltersTimeColumnType.TIME_GRAIN in applied_time_extras: # are there any temporal columns to assign the time grain to? if temporal_columns: applied.append({"column": ExtraFiltersTimeColumnType.TIME_GRAIN}) else: rejected.append( { "reason": ExtraFiltersReasonType.NO_TEMPORAL_COLUMN, "column": ExtraFiltersTimeColumnType.TIME_GRAIN, } ) if applied_time_extras.get(ExtraFiltersTimeColumnType.TIME_RANGE): # are there any temporal columns to assign the time range to? if temporal_columns: applied.append({"column": ExtraFiltersTimeColumnType.TIME_RANGE}) else: rejected.append( { "reason": ExtraFiltersReasonType.NO_TEMPORAL_COLUMN, "column": ExtraFiltersTimeColumnType.TIME_RANGE, } ) return applied, rejected def format_list(items: Sequence[str], sep: str = ", ", quote: str = '"') -> str: quote_escaped = "\\" + quote return sep.join(f"{quote}{x.replace(quote, quote_escaped)}{quote}" for x in items) def find_duplicates(items: Iterable[InputType]) -> list[InputType]: """Find duplicate items in an iterable.""" return [item for item, count in collections.Counter(items).items() if count > 1] def remove_duplicates( items: Iterable[InputType], key: Callable[[InputType], Any] | None = None ) -> list[InputType]: """Remove duplicate items in an iterable.""" if not key: return list(dict.fromkeys(items).keys()) seen = set() result = [] for item in items: item_key = key(item) if item_key not in seen: seen.add(item_key) result.append(item) return result @dataclass class DateColumn: col_label: str timestamp_format: str | None = None offset: int | None = None time_shift: str | None = None timezone: str | None = None # IANA timezone name def __hash__(self) -> int: return hash(self.col_label) def __eq__(self, other: object) -> bool: return isinstance(other, DateColumn) and hash(self) == hash(other) @classmethod def get_legacy_time_column( cls, timestamp_format: str | None, offset: int | None, time_shift: str | None, timezone: str | None = None, ) -> DateColumn: return cls( timestamp_format=timestamp_format, offset=offset, time_shift=time_shift, timezone=timezone, col_label=DTTM_ALIAS, ) def _process_datetime_column( df: pd.DataFrame, col: DateColumn, ) -> None: """Process a single datetime column with format detection.""" if col.timestamp_format in ("epoch_s", "epoch_ms"): dttm_series = df[col.col_label] if is_numeric_dtype(dttm_series): # Column is formatted as a numeric value unit = col.timestamp_format.replace("epoch_", "") df[col.col_label] = pd.to_datetime( dttm_series, utc=False, unit=unit, origin="unix", errors="coerce", exact=False, ) else: # Column has already been formatted as a timestamp. try: df[col.col_label] = dttm_series.apply( lambda x: pd.Timestamp(x) if pd.notna(x) else pd.NaT ) except ValueError: logger.warning( "Unable to convert column %s to datetime, ignoring", col.col_label, ) else: # Try to detect format if not specified format_to_use = col.timestamp_format or detect_datetime_format( df[col.col_label] ) # Parse with or without format (suppress warning if no format) if format_to_use: converted = pd.to_datetime( df[col.col_label], utc=False, format=format_to_use, errors="coerce", exact=False, ) # A format that coerces every non-null value to NaT is a mismatch # (e.g. an epoch-millis column that inherited a '%Y' string format # when used as a chart's granularity). Assigning it would silently # blank the whole column, so keep the original values instead. if df[col.col_label].notna().any() and not converted.notna().any(): logger.warning( "Datetime format %s coerced every value of column %s to NaT; " "keeping the original values", format_to_use, col.col_label, ) else: df[col.col_label] = converted else: with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=".*Could not infer format.*") df[col.col_label] = pd.to_datetime( df[col.col_label], utc=False, format=None, errors="coerce", exact=False, ) def normalize_dttm_col( df: pd.DataFrame, dttm_cols: tuple[DateColumn, ...] = tuple(), # noqa: C408 format_map: dict[str, str] | None = None, ) -> None: """ Normalize datetime columns in a DataFrame. :param df: DataFrame to process :param dttm_cols: Tuple of DateColumn objects to process :param format_map: Optional mapping of column names to datetime formats. When provided, these pre-detected formats are used instead of runtime detection, improving performance and consistency. """ for _col in dttm_cols: if _col.col_label not in df.columns: continue # Use format from format_map if available and not already set if format_map and _col.col_label in format_map and not _col.timestamp_format: _col.timestamp_format = format_map[_col.col_label] _process_datetime_column(df, _col) if _col.timezone and isinstance(_col.timezone, str): try: tz = ZoneInfo(_col.timezone) # Data is stored in UTC, convert to the dataset's configured timezone # First make the datetime UTC-aware, then convert to target timezone series = df[_col.col_label] if not series.empty and series.notna().any(): # Convert UTC to target timezone df[_col.col_label] = ( series.dt.tz_localize("UTC") .dt.tz_convert(tz) .dt.tz_localize(None) # Remove timezone info for display ) except ZoneInfoNotFoundError: logging.warning( "Unknown timezone '%s', falling back to offset", _col.timezone ) if _col.offset: df[_col.col_label] += timedelta(hours=_col.offset) elif _col.offset: df[_col.col_label] += timedelta(hours=_col.offset) if _col.time_shift is not None: df[_col.col_label] += parse_human_timedelta(_col.time_shift) def parse_boolean_string(bool_str: str | None) -> bool: """ Convert a string representation of a true/false value into a boolean >>> parse_boolean_string(None) False >>> parse_boolean_string('false') False >>> parse_boolean_string('true') True >>> parse_boolean_string('False') False >>> parse_boolean_string('True') True >>> parse_boolean_string('foo') False >>> parse_boolean_string('0') False >>> parse_boolean_string('1') True :param bool_str: string representation of a value that is assumed to be boolean :return: parsed boolean value """ if bool_str is None: return False return bool_str.lower() in ("y", "Y", "yes", "True", "t", "true", "On", "on", "1") def apply_max_row_limit( limit: int, server_pagination: bool | None = None, ) -> int: """ Override row limit based on server pagination setting :param limit: requested row limit :param server_pagination: whether server-side pagination is enabled, defaults to None :return: Capped row limit >>> apply_max_row_limit(600000, server_pagination=True) # Server pagination 500000 >>> apply_max_row_limit(600000, server_pagination=False) # No pagination 50000 >>> apply_max_row_limit(5000) # No server_pagination specified 5000 >>> apply_max_row_limit(0) # Zero returns default max limit 50000 """ # pylint: disable=import-outside-toplevel max_limit = ( app.config["TABLE_VIZ_MAX_ROW_SERVER"] if server_pagination else app.config["SQL_MAX_ROW"] ) if limit != 0: return min(max_limit, limit) return max_limit def create_zip(files: dict[str, Any]) -> BytesIO: buf = BytesIO() with ZipFile(buf, "w") as bundle: for filename, contents in files.items(): with bundle.open(filename, "w") as fp: fp.write(contents) buf.seek(0) return buf def check_is_safe_zip(zip_file: ZipFile) -> None: """ Checks whether a ZIP file is safe, raises SupersetException if not. :param zip_file: :return: """ # pylint: disable=import-outside-toplevel uncompress_size = 0 compress_size = 0 for zip_file_element in zip_file.infolist(): if zip_file_element.file_size > app.config["ZIPPED_FILE_MAX_SIZE"]: raise SupersetException("Found file with size above allowed threshold") uncompress_size += zip_file_element.file_size compress_size += zip_file_element.compress_size # Bound the total decompressed size, not just the per-file size, so an # archive of many individually-allowed entries cannot exhaust memory. # Checked inside the loop to fail fast once the running total exceeds # the cap rather than after summing every entry. if uncompress_size > app.config["ZIP_FILE_MAX_TOTAL_SIZE"]: raise SupersetException( "Found total uncompressed size above allowed threshold" ) # Guard the division: a zero compressed size would otherwise raise # ZeroDivisionError instead of a clean error. The total-size cap above # still bounds memory when compress_size is reported as zero. if ( compress_size and uncompress_size / compress_size > app.config["ZIP_FILE_MAX_COMPRESS_RATIO"] ): raise SupersetException("Zip compress ratio above allowed threshold") def remove_extra_adhoc_filters(form_data: dict[str, Any]) -> None: """ Remove filters from slice data that originate from a filter box or native filter """ adhoc_filters = { key: value for key, value in form_data.items() if ADHOC_FILTERS_REGEX.match(key) } for key, value in adhoc_filters.items(): form_data[key] = [ filter_ for filter_ in value or [] if not filter_.get("isExtra") ] def to_int(v: Any, value_if_invalid: int = 0) -> int: try: return int(v) except (ValueError, TypeError): return value_if_invalid def get_query_source_from_request() -> QuerySource | None: if not request or not request.referrer: return None # Match on the referrer's path only, so query-string payloads (e.g. # /explore/?next=/dashboard/1/) cannot misattribute the source. The bare # segment covers legacy /superset/dashboard/ referrers and any # application-root prefix (e.g. /myapp/dashboard/1/). try: referrer_path = urlparse(request.referrer).path except ValueError: # Client-controlled header; e.g. "http://[" raises on the IPv6 # bracket check and must not 500 the query path. return None if "/dashboard/" in referrer_path: return QuerySource.DASHBOARD if "/explore/" in referrer_path: return QuerySource.CHART if "/sqllab/" in referrer_path: return QuerySource.SQL_LAB return None def get_user_agent(database: Database, source: QuerySource | None) -> str: # pylint: disable=import-outside-toplevel source = source or get_query_source_from_request() if user_agent_func := app.config["USER_AGENT_FUNC"]: return user_agent_func(database, source) return DEFAULT_USER_AGENT