diff --git a/UPDATING.md b/UPDATING.md index 22198405ddc..4e5afaadb5f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -64,6 +64,28 @@ Update your environment (virtualenv, Docker base image, CI configuration, etc.) Python 3.11+ before upgrading. The `apache/superset-cache:3.10-slim-trixie` and `py310` Docker image variants are no longer published. +### `from_dttm` and `to_dttm` Jinja template variables removed + +The `{{ from_dttm }}` and `{{ to_dttm }}` Jinja template variables, deprecated since +v5.0, have been removed from the Jinja context and are no longer available in virtual +dataset SQL or custom SQL expressions. + +Replace usages with the `get_time_filter()` function. For example: + +```sql +-- Before +SELECT * FROM tbl +WHERE dttm_col > '{{ from_dttm }}' AND dttm_col < '{{ to_dttm }}' + +-- After +{% set tf = get_time_filter("dttm_col") %} +SELECT * FROM tbl +WHERE dttm_col > {{ tf.from_expr }} AND dttm_col <= {{ tf.to_expr }} +``` + +Note that `from_expr` and `to_expr` are already fully-formatted SQL expressions (e.g. +`TO_TIMESTAMP('2024-01-01', ...)`) — do not wrap them in single quotes. + ### Removed deck.gl JavaScript tooltip/data-mutator controls and ENABLE_JAVASCRIPT_CONTROLS The `ENABLE_JAVASCRIPT_CONTROLS` feature flag and the deck.gl chart controls it gated diff --git a/docs/admin_docs/configuration/sql-templating.mdx b/docs/admin_docs/configuration/sql-templating.mdx index f2c21bcef79..a0059667591 100644 --- a/docs/admin_docs/configuration/sql-templating.mdx +++ b/docs/admin_docs/configuration/sql-templating.mdx @@ -39,8 +39,6 @@ made available in the Jinja context: - `columns`: columns which to group by in the query - `filter`: filters applied in the query -- `from_dttm`: start `datetime` value from the selected time range (`None` if undefined). **Note:** Only available in virtual datasets when a time range filter is applied in Explore/Chart views—not available in standalone SQL Lab queries. (deprecated beginning in version 5.0, use `get_time_filter` instead) -- `to_dttm`: end `datetime` value from the selected time range (`None` if undefined). **Note:** Only available in virtual datasets when a time range filter is applied in Explore/Chart views—not available in standalone SQL Lab queries. (deprecated beginning in version 5.0, use `get_time_filter` instead) - `groupby`: columns which to group by in the query (deprecated) - `metrics`: aggregate expressions in the query - `row_limit`: row limit of the query @@ -49,85 +47,26 @@ made available in the Jinja context: - `time_column`: temporal column of the query (`None` if undefined) - `time_grain`: selected time grain (`None` if undefined) -For example, to add a time range to a virtual dataset, you can write the following: +For example, to add a time range to a virtual dataset, use the `get_time_filter` macro: ```sql +{% set tf = get_time_filter("dttm_col") %} SELECT * FROM tbl -WHERE dttm_col > '{{ from_dttm }}' and dttm_col < '{{ to_dttm }}' +WHERE dttm_col > {{ tf.from_expr }} AND dttm_col <= {{ tf.to_expr }} ``` -You can also use [Jinja's logic](https://jinja.palletsprojects.com/en/2.11.x/templates/#tests) -to make your query robust to clearing the timerange filter: - -```sql -SELECT * -FROM tbl -WHERE ( - {% if from_dttm is not none %} - dttm_col > '{{ from_dttm }}' AND - {% endif %} - {% if to_dttm is not none %} - dttm_col < '{{ to_dttm }}' AND - {% endif %} - 1 = 1 -) -``` - -The `1 = 1` at the end ensures a value is present for the `WHERE` clause even when -the time filter is not set. For many database engines, this could be replaced with `true`. - Note that the Jinja parameters are called within _double_ brackets in the query and with _single_ brackets in the logic blocks. ### Understanding Context Availability -Some Jinja variables like `from_dttm`, `to_dttm`, and `filter` are **only available when a chart or dashboard provides them**. They are populated from: +Some Jinja variables like `filter` are **only available when a chart or dashboard provides them**. They are populated from: - Time range filters applied in Explore/Chart views - Dashboard native filters - Filter components -**These variables are NOT available in standalone SQL Lab queries** because there's no filter context. If you try to use `{{ from_dttm }}` directly in SQL Lab, you'll get an "undefined parameter" error. - -#### Testing Time-Filtered Queries in SQL Lab - -To test queries that use time variables in SQL Lab, you have several options: - -**Option 1: Use Jinja defaults (recommended)** - -```sql -SELECT * -FROM tbl -WHERE dttm_col > '{{ from_dttm | default("2024-01-01", true) }}' - AND dttm_col < '{{ to_dttm | default("2024-12-31", true) }}' -``` - -**Option 2: Use SQL Lab Parameters** - -Set parameters in the SQL Lab UI (Parameters menu): -```json -{ - "from_dttm": "2024-01-01", - "to_dttm": "2024-12-31" -} -``` - -**Option 3: Use `{% set %}` for testing** - -```sql -{% set from_dttm = "2024-01-01" %} -{% set to_dttm = "2024-12-31" %} -SELECT * -FROM tbl -WHERE dttm_col > '{{ from_dttm }}' AND dttm_col < '{{ to_dttm }}' -``` - -:::tip -When you save a SQL Lab query as a virtual dataset and use it in a chart with time filters, -the actual filter values will override any defaults or test values you set. -::: - To add custom functionality to the Jinja context, you need to overload the default Jinja context in your environment by defining the `JINJA_CONTEXT_ADDONS` in your superset configuration (`superset_config.py`). Objects referenced in this dictionary are made available for users to use @@ -479,10 +418,13 @@ Here's a concrete example: ### Time Filter -The `{{ get_time_filter() }}` macro returns the time filter applied to a specific column. This is useful if you want -to handle time filters inside the virtual dataset, as by default the time filter is placed on the outer query. This can -considerably improve performance, as many databases and query engines are able to optimize the query better -if the temporal filter is placed on the inner query, as opposed to the outer query. +The `{{ get_time_filter() }}` macro is the standard way to reference the selected time range +in a virtual dataset. It returns a `TimeFilter` object whose `from_expr` and `to_expr` +properties are fully-formatted SQL expressions for the start and end of the range — use them +directly in your `WHERE` clause without quoting. + +Placing the filter inside the virtual dataset (rather than relying on it being applied to +the outer query) also gives many database engines the opportunity to optimize the query better. The macro takes the following parameters: diff --git a/docs/docs/using-superset/sql-templating.mdx b/docs/docs/using-superset/sql-templating.mdx index 44566604565..2e924e7c659 100644 --- a/docs/docs/using-superset/sql-templating.mdx +++ b/docs/docs/using-superset/sql-templating.mdx @@ -146,7 +146,9 @@ The `where_in` filter converts the list to SQL format: `('value1', 'value2', 'va ### Time Filters -For charts with time range filters: +`get_time_filter()` is the standard way to reference the selected time range in a virtual dataset. +It returns a `TimeFilter` object with `from_expr` and `to_expr` — fully-formatted SQL expressions +that should be used directly in `WHERE` clauses without quoting. | Macro | Description | |-------|-------------| @@ -198,17 +200,9 @@ GROUP BY category ## Testing Templates in SQL Lab -Some variables like `from_dttm` and `filter_values()` only work when filters are applied from dashboards or charts. To test in SQL Lab: +Some variables like `filter_values()` only work when filters are applied from dashboards or charts. To test in SQL Lab: -**Option 1: Use defaults** - -```sql -SELECT * -FROM orders -WHERE date >= '{{ from_dttm | default("2024-01-01", true) }}' -``` - -**Option 2: Set test parameters** +**Option 1: Set test parameters** Add to the Parameters menu: @@ -220,7 +214,7 @@ Add to the Parameters menu: } ``` -**Option 3: Use `{% set %}`** +**Option 2: Use `{% set %}`** ```sql {% set start_date = "2024-01-01" %} diff --git a/superset/jinja_context.py b/superset/jinja_context.py index 025b0cb0a61..806a82280c7 100644 --- a/superset/jinja_context.py +++ b/superset/jinja_context.py @@ -25,7 +25,6 @@ from datetime import datetime from functools import lru_cache, partial from typing import Any, Callable, cast, TYPE_CHECKING, TypedDict, Union -import dateutil from flask import current_app, g, has_request_context, request from flask_babel import gettext as _ from jinja2 import DebugUndefined, Environment, TemplateSyntaxError, UndefinedError @@ -887,19 +886,6 @@ class BaseTemplateProcessor: class JinjaTemplateProcessor(BaseTemplateProcessor): - def _parse_datetime(self, dttm: str) -> datetime | None: - """ - Try to parse a datetime and default to None in the worst case. - - Since this may have been rendered by different engines, the datetime may - vary slightly in format. We try to make it consistent, and if all else - fails, just return None. - """ - try: - return dateutil.parser.parse(dttm) - except dateutil.parser.ParserError: - return None - def set_context(self, **kwargs: Any) -> None: super().set_context(**kwargs) extra_cache = ExtraCache( @@ -912,23 +898,6 @@ class JinjaTemplateProcessor(BaseTemplateProcessor): query_context_filters=self._context.get("filter") or [], ) - from_dttm = ( - self._parse_datetime(dttm) - if (dttm := self._context.get("from_dttm")) - else None - ) - to_dttm = ( - self._parse_datetime(dttm) - if (dttm := self._context.get("to_dttm")) - else None - ) - - dataset_macro_with_context = partial( - dataset_macro, - from_dttm=from_dttm, - to_dttm=to_dttm, - ) - self._context.update( { "url_param": partial(safe_proxy, extra_cache.url_param), @@ -946,7 +915,7 @@ class JinjaTemplateProcessor(BaseTemplateProcessor): "cache_key_wrapper": partial(safe_proxy, extra_cache.cache_key_wrapper), "filter_values": partial(safe_proxy, extra_cache.filter_values), "get_filters": partial(safe_proxy, extra_cache.get_filters), - "dataset": partial(safe_proxy, dataset_macro_with_context), + "dataset": partial(safe_proxy, dataset_macro), "get_time_filter": partial(safe_proxy, extra_cache.get_time_filter), } ) @@ -1104,18 +1073,12 @@ def dataset_macro( dataset_id: int, include_metrics: bool = False, columns: list[str] | None = None, - from_dttm: datetime | None = None, - to_dttm: datetime | None = None, ) -> str: """ Given a dataset ID, return the SQL that represents it. The generated SQL includes all columns (including computed) by default. Optionally the user can also request metrics to be included, and columns to group by. - - The from_dttm and to_dttm parameters are filled in from filter values in explore - views, and we take them to make those properties available to jinja templates in - the underlying dataset. """ # pylint: disable=import-outside-toplevel from superset.daos.dataset import DatasetDAO @@ -1131,8 +1094,8 @@ def dataset_macro( "filter": [], "metrics": metrics if include_metrics else None, "columns": cast(list[Column], columns), - "from_dttm": from_dttm, - "to_dttm": to_dttm, + "from_dttm": None, + "to_dttm": None, } sqla_query = dataset.get_query_str_extended(query_obj, mutate=False) sql = sqla_query.sql diff --git a/superset/models/helpers.py b/superset/models/helpers.py index ab3664cd15e..f515b383c44 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -3530,14 +3530,12 @@ class ExploreMixin: # pylint: disable=too-many-public-methods template_kwargs = { "columns": columns, - "from_dttm": from_dttm.isoformat() if from_dttm else None, "groupby": groupby, "metrics": metrics, "row_limit": row_limit, "row_offset": row_offset, "time_column": granularity, "time_grain": time_grain, - "to_dttm": to_dttm.isoformat() if to_dttm else None, "table_columns": [col.column_name for col in self.columns], "filter": filter, } diff --git a/tests/unit_tests/connectors/sqla/utils_test.py b/tests/unit_tests/connectors/sqla/utils_test.py index d74b4d94e79..dbcbd2ce1e5 100644 --- a/tests/unit_tests/connectors/sqla/utils_test.py +++ b/tests/unit_tests/connectors/sqla/utils_test.py @@ -219,15 +219,15 @@ def test_get_virtual_table_metadata_renders_jinja(mocker: MockerFixture) -> None be rendered via the template processor before SQL parsing. Otherwise the raw Jinja tokens reach sqlglot and the parser rejects them as a syntax error (the user-visible symptom is "Invalid SQL" when clicking - "SYNC COLUMNS FROM SOURCE" on a dataset that uses {{ from_dttm }} etc.). + "SYNC COLUMNS FROM SOURCE" on a dataset that uses Jinja macros). """ mock_get_columns_description = mocker.patch( "superset.connectors.sqla.utils.get_columns_description", return_value=[{"name": "rendered_col", "type": "INTEGER"}], ) - raw_sql = "SELECT * FROM tbl WHERE ts > '{{ from_dttm }}'" - rendered_sql = "SELECT * FROM tbl WHERE ts > '2024-01-01 00:00:00'" + raw_sql = "SELECT * FROM tbl WHERE user_id = {{ current_user_id() }}" + rendered_sql = "SELECT * FROM tbl WHERE user_id = 42" dataset = mocker.MagicMock(sql=raw_sql) dataset.database.db_engine_spec.engine = "postgresql"