From c6c9114b40ee9711668008fa13a3ed947c5d0976 Mon Sep 17 00:00:00 2001 From: SBIN2010 <132096459+SBIN2010@users.noreply.github.com> Date: Mon, 24 Mar 2025 19:48:39 +0300 Subject: [PATCH 01/12] fix: CSV/Excel upload form change column dates description (#32797) --- .../src/features/databases/UploadDataModel/index.tsx | 2 +- superset/translations/ar/LC_MESSAGES/messages.po | 4 ++-- superset/translations/de/LC_MESSAGES/messages.po | 6 +++--- superset/translations/en/LC_MESSAGES/messages.po | 2 +- superset/translations/es/LC_MESSAGES/messages.po | 4 ++-- superset/translations/fr/LC_MESSAGES/messages.po | 6 +++--- superset/translations/it/LC_MESSAGES/messages.po | 4 ++-- superset/translations/ja/LC_MESSAGES/messages.po | 4 ++-- superset/translations/ko/LC_MESSAGES/messages.po | 4 ++-- superset/translations/messages.pot | 2 +- superset/translations/nl/LC_MESSAGES/messages.po | 6 +++--- superset/translations/pl/LC_MESSAGES/messages.po | 4 ++-- superset/translations/pt/LC_MESSAGES/messages.po | 4 ++-- superset/translations/pt_BR/LC_MESSAGES/messages.po | 6 +++--- superset/translations/ru/LC_MESSAGES/messages.po | 6 +++--- superset/translations/sk/LC_MESSAGES/messages.po | 2 +- superset/translations/sl/LC_MESSAGES/messages.po | 4 ++-- superset/translations/tr/LC_MESSAGES/messages.po | 4 ++-- superset/translations/uk/LC_MESSAGES/messages.po | 4 ++-- superset/translations/zh/LC_MESSAGES/messages.po | 4 ++-- superset/translations/zh_TW/LC_MESSAGES/messages.po | 4 ++-- 21 files changed, 43 insertions(+), 43 deletions(-) diff --git a/superset-frontend/src/features/databases/UploadDataModel/index.tsx b/superset-frontend/src/features/databases/UploadDataModel/index.tsx index 8235049283d..39dd14f82d4 100644 --- a/superset-frontend/src/features/databases/UploadDataModel/index.tsx +++ b/superset-frontend/src/features/databases/UploadDataModel/index.tsx @@ -793,7 +793,7 @@ const UploadDataModal: FunctionComponent = ({ allowClear allowNewOptions placeholder={t( - 'A comma separated list of columns that should be parsed as dates', + 'Select column names from a dropdown list that should be parsed as dates.', )} /> diff --git a/superset/translations/ar/LC_MESSAGES/messages.po b/superset/translations/ar/LC_MESSAGES/messages.po index 8fd18603d46..38e6732d0e6 100644 --- a/superset/translations/ar/LC_MESSAGES/messages.po +++ b/superset/translations/ar/LC_MESSAGES/messages.po @@ -630,8 +630,8 @@ msgstr ">= (أكبر أو يساوي)" msgid "A Big Number" msgstr "عدد كبير" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "قائمة مفصولة بفواصل من الأعمدة التي يجب تحليلها كتواريخ" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "حدد أسماء الأعمدة من القائمة المنسدلة التي يجب معاملتها كتواريخ." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "قائمة مفصولة بفواصل من المخططات التي يسمح للملفات بالتحميل إليها." diff --git a/superset/translations/de/LC_MESSAGES/messages.po b/superset/translations/de/LC_MESSAGES/messages.po index a6b8ba611da..81c65007d6d 100644 --- a/superset/translations/de/LC_MESSAGES/messages.po +++ b/superset/translations/de/LC_MESSAGES/messages.po @@ -664,10 +664,10 @@ msgstr ">= (Größer oder gleich)" msgid "A Big Number" msgstr "Eine Große Zahl" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Eine durch Kommas getrennte Liste von Spalten, die als Datumsangaben " -"interpretiert werden sollen" +"Wählen Sie aus der Dropdown-Liste die Namen der Spalten aus, die als Datum " +"interpretiert werden sollen." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/en/LC_MESSAGES/messages.po b/superset/translations/en/LC_MESSAGES/messages.po index e9802f1b7e6..8b060e4a208 100644 --- a/superset/translations/en/LC_MESSAGES/messages.po +++ b/superset/translations/en/LC_MESSAGES/messages.po @@ -579,7 +579,7 @@ msgstr "" msgid "A Big Number" msgstr "" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/es/LC_MESSAGES/messages.po b/superset/translations/es/LC_MESSAGES/messages.po index 88b49fb2828..748d1fa7ddf 100644 --- a/superset/translations/es/LC_MESSAGES/messages.po +++ b/superset/translations/es/LC_MESSAGES/messages.po @@ -618,9 +618,9 @@ msgid "A Big Number" msgstr "Número Grande" #, fuzzy -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Una lista separada por comas de columnas que deben ser parseadas como " +"Seleccione en la lista desplegable los nombres de las columnas que deben analizarse como " "fechas." #, fuzzy diff --git a/superset/translations/fr/LC_MESSAGES/messages.po b/superset/translations/fr/LC_MESSAGES/messages.po index b8724e187af..637dc77e96f 100644 --- a/superset/translations/fr/LC_MESSAGES/messages.po +++ b/superset/translations/fr/LC_MESSAGES/messages.po @@ -624,10 +624,10 @@ msgstr ">= (plus grand ou égal)" msgid "A Big Number" msgstr "Gros nombre" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Une liste de colonnes séparées par des virgules qui doivent être analysées comme " -"des dates" +"Sélectionnez dans la liste déroulante les noms des colonnes à analyser en tant " +"que dates" #, fuzzy msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/it/LC_MESSAGES/messages.po b/superset/translations/it/LC_MESSAGES/messages.po index 8458fd95542..001dc63c114 100644 --- a/superset/translations/it/LC_MESSAGES/messages.po +++ b/superset/translations/it/LC_MESSAGES/messages.po @@ -601,8 +601,8 @@ msgstr "" msgid "A Big Number" msgstr "Numero Grande" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "Selezionare i nomi delle colonne da elaborare come date dall'elenco a discesa." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/ja/LC_MESSAGES/messages.po b/superset/translations/ja/LC_MESSAGES/messages.po index f8970831184..fd9ee19b8ed 100644 --- a/superset/translations/ja/LC_MESSAGES/messages.po +++ b/superset/translations/ja/LC_MESSAGES/messages.po @@ -616,8 +616,8 @@ msgstr ">= (大きいか等しい)" msgid "A Big Number" msgstr "大きな数字" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "日付として解析する必要がある列のカンマ区切りのリスト" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "日付として解析する列の名前をドロップダウン・リストから選択する。" msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "ファイルのアップロードを許可するスキーマのカンマ区切りのリスト" diff --git a/superset/translations/ko/LC_MESSAGES/messages.po b/superset/translations/ko/LC_MESSAGES/messages.po index 2c8f1a8f7d4..bd245cfa076 100644 --- a/superset/translations/ko/LC_MESSAGES/messages.po +++ b/superset/translations/ko/LC_MESSAGES/messages.po @@ -599,8 +599,8 @@ msgstr "" msgid "A Big Number" msgstr "테이블 명" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "드롭다운 목록에서 날짜로 처리할 열 이름을 선택합니다." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot index fc9f1c937dc..e32e2b65976 100644 --- a/superset/translations/messages.pot +++ b/superset/translations/messages.pot @@ -585,7 +585,7 @@ msgstr "" msgid "A Big Number" msgstr "" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/nl/LC_MESSAGES/messages.po b/superset/translations/nl/LC_MESSAGES/messages.po index b7717a70943..e652b9b74f4 100644 --- a/superset/translations/nl/LC_MESSAGES/messages.po +++ b/superset/translations/nl/LC_MESSAGES/messages.po @@ -629,10 +629,10 @@ msgstr ">= (Groter of gelijk)" msgid "A Big Number" msgstr "Een groot getal" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Een door komma's gescheiden lijst van kolommen die als datums moeten " -"worden geïnterpreteerd" +"Selecteer de kolomnamen die moeten worden behandeld als datums " +"in de vervolgkeuzelijst" msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/pl/LC_MESSAGES/messages.po b/superset/translations/pl/LC_MESSAGES/messages.po index 0c22650fb00..9aca1ac01b1 100644 --- a/superset/translations/pl/LC_MESSAGES/messages.po +++ b/superset/translations/pl/LC_MESSAGES/messages.po @@ -646,8 +646,8 @@ msgid "A Big Number" msgstr "Duża liczba" #, fuzzy -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "Lista kolumn oddzielonych przecinkami, które powinny być traktowane jako daty." +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "Z listy rozwijanej wybierz nazwy kolumn, które mają być traktowane jako daty." #, fuzzy msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/pt/LC_MESSAGES/messages.po b/superset/translations/pt/LC_MESSAGES/messages.po index ecd7db908fc..32e5bc7a54a 100644 --- a/superset/translations/pt/LC_MESSAGES/messages.po +++ b/superset/translations/pt/LC_MESSAGES/messages.po @@ -606,8 +606,8 @@ msgstr "" msgid "A Big Number" msgstr "Número grande" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "Selecione os nomes das colunas a serem tratadas como datas na lista pendente." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/pt_BR/LC_MESSAGES/messages.po b/superset/translations/pt_BR/LC_MESSAGES/messages.po index ab06eb013ac..6aec74f7517 100644 --- a/superset/translations/pt_BR/LC_MESSAGES/messages.po +++ b/superset/translations/pt_BR/LC_MESSAGES/messages.po @@ -642,10 +642,10 @@ msgstr ">= (Maior ou equal)" msgid "A Big Number" msgstr "Um grande número" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Uma lista separada por vírgulas de colunas que devem ser analisadas como " -"datas" +"Selecione os nomes das colunas a serem analisadas como datas na lista " +"pendente" msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/ru/LC_MESSAGES/messages.po b/superset/translations/ru/LC_MESSAGES/messages.po index e37055b24e5..a4e5996d0a3 100644 --- a/superset/translations/ru/LC_MESSAGES/messages.po +++ b/superset/translations/ru/LC_MESSAGES/messages.po @@ -645,10 +645,10 @@ msgstr ">= (больше или равно)" msgid "A Big Number" msgstr "Карточка" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" -"Разделенный запятыми список столбцов, которые должны быть " -"интерпретированы как даты." +"Выберите из выпадающего списка имена столбцов, которые должны быть " +"обработаны как даты" msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "Разделенный запятыми список схем, в которые можно загружать файлы." diff --git a/superset/translations/sk/LC_MESSAGES/messages.po b/superset/translations/sk/LC_MESSAGES/messages.po index c92d3083d11..d1ce35080bd 100644 --- a/superset/translations/sk/LC_MESSAGES/messages.po +++ b/superset/translations/sk/LC_MESSAGES/messages.po @@ -583,7 +583,7 @@ msgstr "" msgid "A Big Number" msgstr "" -msgid "A comma separated list of columns that should be parsed as dates" +msgid "Select column names from a dropdown list that should be parsed as dates." msgstr "" msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/sl/LC_MESSAGES/messages.po b/superset/translations/sl/LC_MESSAGES/messages.po index 55cc6c0a086..d0ee1045446 100644 --- a/superset/translations/sl/LC_MESSAGES/messages.po +++ b/superset/translations/sl/LC_MESSAGES/messages.po @@ -681,8 +681,8 @@ msgstr ">= (večje ali enako)" msgid "A Big Number" msgstr "Velika številka" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "Z vejico ločen seznam stolpcev, v katerih bodo prepoznani datumi" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "S spustnega seznama izberite imena stolpcev, ki bodo obravnavani kot datumi." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "Z vejicami ločen seznam shem, kjer je dovoljeno nalaganje datotek." diff --git a/superset/translations/tr/LC_MESSAGES/messages.po b/superset/translations/tr/LC_MESSAGES/messages.po index 0fadea94f0d..c51c4b1e910 100644 --- a/superset/translations/tr/LC_MESSAGES/messages.po +++ b/superset/translations/tr/LC_MESSAGES/messages.po @@ -581,8 +581,8 @@ msgstr "" msgid "A Big Number" msgstr "" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "Açılır listeden tarih olarak değerlendirilecek sütun adlarını seçin." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "" diff --git a/superset/translations/uk/LC_MESSAGES/messages.po b/superset/translations/uk/LC_MESSAGES/messages.po index e62117c9a59..235335e2f87 100644 --- a/superset/translations/uk/LC_MESSAGES/messages.po +++ b/superset/translations/uk/LC_MESSAGES/messages.po @@ -634,8 +634,8 @@ msgstr "> = (Більший або рівний)" msgid "A Big Number" msgstr "Велика кількість" -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "Кома -розділений список стовпців, які слід проаналізувати як дати" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "Виберіть імена стовпців зі списку, що випадає, які слід розібрати як дати." msgid "A comma-separated list of schemas that files are allowed to upload to." msgstr "Список схем, відокремлений комами, до яких файли дозволяють завантажувати." diff --git a/superset/translations/zh/LC_MESSAGES/messages.po b/superset/translations/zh/LC_MESSAGES/messages.po index 9b96fb82365..4128f99c909 100644 --- a/superset/translations/zh/LC_MESSAGES/messages.po +++ b/superset/translations/zh/LC_MESSAGES/messages.po @@ -621,8 +621,8 @@ msgid "A Big Number" msgstr "大数字" #, fuzzy -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "应作为日期解析的列的逗号分隔列表。" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "从下拉列表中选择要作为日期处理的列名称。" #, fuzzy msgid "A comma-separated list of schemas that files are allowed to upload to." diff --git a/superset/translations/zh_TW/LC_MESSAGES/messages.po b/superset/translations/zh_TW/LC_MESSAGES/messages.po index a76e95acd1d..1fe130e01f6 100644 --- a/superset/translations/zh_TW/LC_MESSAGES/messages.po +++ b/superset/translations/zh_TW/LC_MESSAGES/messages.po @@ -620,8 +620,8 @@ msgid "A Big Number" msgstr "大數字" #, fuzzy -msgid "A comma separated list of columns that should be parsed as dates" -msgstr "應作為日期解析的列的逗號分隔列表。" +msgid "Select column names from a dropdown list that should be parsed as dates." +msgstr "从下拉列表中选择要分析的日期列名称。" #, fuzzy msgid "A comma-separated list of schemas that files are allowed to upload to." From cedd186c21e46df134bdfd2cf4c8df753977f1b6 Mon Sep 17 00:00:00 2001 From: Vitor Avila <96086495+Vitor-Avila@users.noreply.github.com> Date: Mon, 24 Mar 2025 16:55:37 -0300 Subject: [PATCH 02/12] feat(Jinja): to_datetime filter (#32781) --- docs/docs/configuration/sql-templating.mdx | 34 +++++++++++++ superset/jinja_context.py | 19 ++++++++ tests/unit_tests/jinja_context_test.py | 55 ++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/docs/docs/configuration/sql-templating.mdx b/docs/docs/configuration/sql-templating.mdx index 0e618fd9c19..7f094f1d87a 100644 --- a/docs/docs/configuration/sql-templating.mdx +++ b/docs/docs/configuration/sql-templating.mdx @@ -461,3 +461,37 @@ This macro avoids copy/paste, allowing users to centralize the metric definition The `dataset_id` parameter is optional, and if not provided Superset will use the current dataset from context (for example, when using this macro in the Chart Builder, by default the `macro_key` will be searched in the dataset powering the chart). The parameter can be used in SQL Lab, or when fetching a metric from another dataset. + +## Available Filters + +Superset supports [builtin filters from the Jinja2 templating package](https://jinja.palletsprojects.com/en/stable/templates/#builtin-filters). Custom filters have also been implemented: + +**Where In** +Parses a list into a SQL-compatible statement. This is useful with macros that return an array (for example the `filter_values` macro): + +``` +Dashboard filter with "First", "Second" and "Third" options selected +{{ filter_values('column') }} => ["First", "Second", "Third"] +{{ filter_values('column')|where_in }} => ('First', 'Second', 'Third') +``` + +By default, this filter returns `()` (as a string) in case the value is null. The `default_to_none` parameter can be se to `True` to return null in this case: + +``` +Dashboard filter without any value applied +{{ filter_values('column') }} => () +{{ filter_values('column')|where_in(default_to_none=True) }} => None +``` + +**To Datetime** + +Loads a string as a `datetime` object. This is useful when performing date operations. For example: +``` +{% set from_expr = get_time_filter("dttm", strftime="%Y-%m-%d").from_expr %} +{% set to_expr = get_time_filter("dttm", strftime="%Y-%m-%d").to_expr %} +{% if (to_expr|to_datetime(format="%Y-%m-%d") - from_expr|to_datetime(format="%Y-%m-%d")).days > 100 %} + do something +{% else %} + do something else +{% endif %} +``` diff --git a/superset/jinja_context.py b/superset/jinja_context.py index a4d8e6d6a49..0f56886dd2e 100644 --- a/superset/jinja_context.py +++ b/superset/jinja_context.py @@ -561,6 +561,24 @@ class WhereInMacro: # pylint: disable=too-few-public-methods return result +def to_datetime( + value: str | None, format: str = "%Y-%m-%d %H:%M:%S" +) -> datetime | None: + """ + Parses a string into a datetime object. + + :param value: the string to parse. + :param format: the format to parse the string with. + :returns: the parsed datetime object. + """ + if not value: + return None + + # This value might come from a macro that could be including wrapping quotes + value = value.strip("'\"") + return datetime.strptime(value, format) + + class BaseTemplateProcessor: """ Base class for database-specific jinja context @@ -596,6 +614,7 @@ class BaseTemplateProcessor: # custom filters self.env.filters["where_in"] = WhereInMacro(database.get_dialect()) + self.env.filters["to_datetime"] = to_datetime def set_context(self, **kwargs: Any) -> None: self._context.update(kwargs) diff --git a/tests/unit_tests/jinja_context_test.py b/tests/unit_tests/jinja_context_test.py index be6e5fb55e3..5cc6f218a89 100644 --- a/tests/unit_tests/jinja_context_test.py +++ b/tests/unit_tests/jinja_context_test.py @@ -17,6 +17,7 @@ # pylint: disable=invalid-name, unused-argument from __future__ import annotations +from datetime import datetime from typing import Any import pytest @@ -38,6 +39,7 @@ from superset.jinja_context import ( metric_macro, safe_proxy, TimeFilter, + to_datetime, WhereInMacro, ) from superset.models.core import Database @@ -429,6 +431,59 @@ def test_where_in_empty_list() -> None: assert where_in([], default_to_none=True) is None +@pytest.mark.parametrize( + "value,format,output", + [ + ("2025-03-20 15:55:00", None, datetime(2025, 3, 20, 15, 55)), + (None, None, None), + ("2025-03-20", "%Y-%m-%d", datetime(2025, 3, 20)), + ("'2025-03-20'", "%Y-%m-%d", datetime(2025, 3, 20)), + ], +) +def test_to_datetime( + value: str | None, format: str | None, output: datetime | None +) -> None: + """ + Test the ``to_datetime`` custom filter. + """ + + result = ( + to_datetime(value, format=format) if format is not None else to_datetime(value) + ) + assert result == output + + +@pytest.mark.parametrize( + "value,format,match", + [ + ( + "2025-03-20", + None, + "time data '2025-03-20' does not match format '%Y-%m-%d %H:%M:%S'", + ), + ( + "2025-03-20 15:55:00", + "%Y-%m-%d", + "unconverted data remains: 15:55:00", + ), + ], +) +def test_to_datetime_raises(value: str, format: str | None, match: str) -> None: + """ + Test the ``to_datetime`` custom filter raises with an incorrect + format. + """ + with pytest.raises( + ValueError, + match=match, + ): + ( + to_datetime(value, format=format) + if format is not None + else to_datetime(value) + ) + + def test_dataset_macro(mocker: MockerFixture) -> None: """ Test the ``dataset_macro`` macro. From b339d7ad2075222ba38bbe289f3c1ea6584c5dd6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:43:01 -0700 Subject: [PATCH 03/12] =?UTF-8?q?chore(=F0=9F=A6=BE):=20bump=20python=20cl?= =?UTF-8?q?ick-option-group=200.5.6=20->=200.5.7=20(#32825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: GitHub Action --- requirements/base.txt | 2 +- requirements/development.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index e3a92219e1a..2b0694c9c92 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -68,7 +68,7 @@ click==8.1.8 # flask-appbuilder click-didyoumean==0.3.1 # via celery -click-option-group==0.5.6 +click-option-group==0.5.7 # via apache-superset (pyproject.toml) click-plugins==1.1.1 # via celery diff --git a/requirements/development.txt b/requirements/development.txt index 68ec6a17c37..7a71b075db5 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -107,7 +107,7 @@ click-didyoumean==0.3.1 # via # -c requirements/base.txt # celery -click-option-group==0.5.6 +click-option-group==0.5.7 # via # -c requirements/base.txt # apache-superset From fd4e45aafc9f8dc2249199c3961e2d2ce8a535fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:43:23 -0700 Subject: [PATCH 04/12] =?UTF-8?q?chore(=F0=9F=A6=BE):=20bump=20python=20sh?= =?UTF-8?q?illelagh=20subpackage(s)=20(#32828)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: GitHub Action --- requirements/base.txt | 2 +- requirements/development.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 2b0694c9c92..f7b89a5be56 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -264,7 +264,7 @@ parsedatetime==2.6 # via apache-superset (pyproject.toml) pgsanity==0.2.9 # via apache-superset (pyproject.toml) -platformdirs==4.3.6 +platformdirs==4.3.7 # via requests-cache ply==3.11 # via jsonpath-ng diff --git a/requirements/development.txt b/requirements/development.txt index 7a71b075db5..6305ea82718 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -543,7 +543,7 @@ pillow==10.3.0 # via # apache-superset # matplotlib -platformdirs==4.3.6 +platformdirs==4.3.7 # via # -c requirements/base.txt # requests-cache From 7bc349c3c3eae93217bb37a986843e7826032bf1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:43:52 -0700 Subject: [PATCH 05/12] =?UTF-8?q?chore(=F0=9F=A6=BE):=20bump=20python=20py?= =?UTF-8?q?parsing=203.2.1=20->=203.2.2=20(#32827)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: GitHub Action --- requirements/base.txt | 2 +- requirements/development.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index f7b89a5be56..fe2fba28af6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -295,7 +295,7 @@ pynacl==1.5.0 # via paramiko pyopenssl==25.0.0 # via shillelagh -pyparsing==3.2.1 +pyparsing==3.2.2 # via apache-superset (pyproject.toml) pysocks==1.7.1 # via urllib3 diff --git a/requirements/development.txt b/requirements/development.txt index 6305ea82718..49432846a0d 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -636,7 +636,7 @@ pyopenssl==25.0.0 # via # -c requirements/base.txt # shillelagh -pyparsing==3.2.1 +pyparsing==3.2.2 # via # -c requirements/base.txt # apache-superset From b7435f84f055d0f1379ebc78c6dfc30c69e976cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:44:15 -0700 Subject: [PATCH 06/12] =?UTF-8?q?chore(=F0=9F=A6=BE):=20bump=20python=20hu?= =?UTF-8?q?manize=204.12.1=20->=204.12.2=20(#32826)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: GitHub Action --- requirements/base.txt | 2 +- requirements/development.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index fe2fba28af6..3923bc2278e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -167,7 +167,7 @@ hashids==1.3.1 # via apache-superset (pyproject.toml) holidays==0.25 # via apache-superset (pyproject.toml) -humanize==4.12.1 +humanize==4.12.2 # via apache-superset (pyproject.toml) idna==3.10 # via diff --git a/requirements/development.txt b/requirements/development.txt index 49432846a0d..86be03515c8 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -343,7 +343,7 @@ holidays==0.25 # -c requirements/base.txt # apache-superset # prophet -humanize==4.12.1 +humanize==4.12.2 # via # -c requirements/base.txt # apache-superset From 6b96b37c3831375a84203a81794ead84d9ca6577 Mon Sep 17 00:00:00 2001 From: bmaquet Date: Mon, 24 Mar 2025 22:39:07 +0100 Subject: [PATCH 07/12] feat: Add current_user_roles() Jinja macro (#32770) --- docs/docs/configuration/sql-templating.mdx | 30 ++++++++++++++++++++ superset/jinja_context.py | 19 +++++++++++++ superset/utils/core.py | 13 +++++++++ tests/integration_tests/sqla_models_tests.py | 17 +++++++---- tests/unit_tests/jinja_context_test.py | 9 +++++- 5 files changed, 82 insertions(+), 6 deletions(-) diff --git a/docs/docs/configuration/sql-templating.mdx b/docs/docs/configuration/sql-templating.mdx index 7f094f1d87a..dcd5bb0869b 100644 --- a/docs/docs/configuration/sql-templating.mdx +++ b/docs/docs/configuration/sql-templating.mdx @@ -220,6 +220,36 @@ cache key by adding the following parameter to your Jinja code: {{ current_user_email(add_to_cache_keys=False) }} ``` +**Current User Roles** + +The `{{ current_user_roles() }}` macro returns an array of roles for the logged in user. + +If you have caching enabled in your Superset configuration, then by default the roles value will be used +by Superset when calculating the cache key. A cache key is a unique identifier that determines if there's a +cache hit in the future and Superset can retrieve cached data. + +You can disable the inclusion of the roles value in the calculation of the +cache key by adding the following parameter to your Jinja code: + +```python +{{ current_user_roles(add_to_cache_keys=False) }} +``` + +You can json-stringify the array by adding `|tojson` to your Jinja code: +```python +{{ current_user_roles()|tojson }} +``` + +You can use the `|where_in` filter to use your roles in a SQL statement. For example, if `current_user_roles()` returns `['admin', 'viewer']`, the following template: +```python +SELECT * FROM users WHERE role IN {{ current_user_roles()|where_in }} +``` + +Will be rendered as: +```sql +SELECT * FROM users WHERE role IN ('admin', 'viewer') +``` + **Custom URL Parameters** The `{{ url_param('custom_variable') }}` macro lets you define arbitrary URL diff --git a/superset/jinja_context.py b/superset/jinja_context.py index 0f56886dd2e..c32d097b5bf 100644 --- a/superset/jinja_context.py +++ b/superset/jinja_context.py @@ -46,6 +46,7 @@ from superset.utils.core import ( FilterOperator, get_user_email, get_user_id, + get_user_roles, get_username, merge_extra_filters, ) @@ -108,6 +109,7 @@ class ExtraCache: r"current_user_id\([^()]*\)|" r"current_username\([^()]*\)|" r"current_user_email\([^()]*\)|" + r"current_user_roles\([^()]*\)|" r"cache_key_wrapper\([^()]*\)|" r"url_param\([^()]*\)" r")" @@ -172,6 +174,20 @@ class ExtraCache: return email_address return None + def current_user_roles(self, add_to_cache_keys: bool = True) -> list[str] | None: + """ + Return the list of roles of the user who is currently logged in. + + :param add_to_cache_keys: Whether the value should be included in the cache key + :returns: List of role names + """ + + if user_roles := get_user_roles(): + if add_to_cache_keys: + self.cache_key_wrapper(json.dumps(user_roles)) + return user_roles + return None + def cache_key_wrapper(self, key: Any) -> Any: """ Adds values to a list that is added to the query object used for calculating a @@ -689,6 +705,9 @@ class JinjaTemplateProcessor(BaseTemplateProcessor): "current_user_email": partial( safe_proxy, extra_cache.current_user_email ), + "current_user_roles": partial( + safe_proxy, extra_cache.current_user_roles + ), "cache_key_wrapper": partial(safe_proxy, extra_cache.cache_key_wrapper), "filter_values": partial(safe_proxy, extra_cache.filter_values), "get_filters": partial(safe_proxy, extra_cache.get_filters), diff --git a/superset/utils/core.py b/superset/utils/core.py index 2b80c89f612..6caea19b1e2 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -1292,6 +1292,19 @@ def get_user_email() -> str | None: return None +def get_user_roles() -> list[str] | None: + """ + Get the roles (if defined) associated with the current user. + + :returns: The sorted list of roles + """ + + try: + return sorted([role.name for role in g.user.roles]) + except Exception: # pylint: disable=broad-except + return None + + @contextmanager def override_user(user: User | None, force: bool = True) -> Iterator[Any]: """ diff --git a/tests/integration_tests/sqla_models_tests.py b/tests/integration_tests/sqla_models_tests.py index c3f4328bd66..f19d495fd87 100644 --- a/tests/integration_tests/sqla_models_tests.py +++ b/tests/integration_tests/sqla_models_tests.py @@ -797,9 +797,10 @@ def test_none_operand_in_filter(login_as_admin, physical_dataset): SELECT '{{ current_user_id() }}' as id, '{{ current_username() }}' as username, - '{{ current_user_email() }}' as email + '{{ current_user_email() }}' as email, + '{{ current_user_roles()|tojson }}' as roles """, - {1, "abc", "abc@test.com"}, + {1, "abc", "abc@test.com", '["role1", "role2"]'}, True, ), ( @@ -809,9 +810,10 @@ def test_none_operand_in_filter(login_as_admin, physical_dataset): SELECT '{{ current_user_id() }}' as id, '{{ current_username() }}' as username, - '{{ user_email }}' as email + '{{ user_email }}' as email, + '{{ current_user_roles()|tojson }}' as roles """, - {1, "abc", "abc@test.com"}, + {1, "abc", "abc@test.com", '["role1", "role2"]'}, True, ), ( @@ -830,7 +832,8 @@ def test_none_operand_in_filter(login_as_admin, physical_dataset): SELECT '{{ current_user_id(False) }}' as id, '{{ current_username(False) }}' as username, - '{{ current_user_email(False) }}' as email + '{{ current_user_email(False) }}' as email, + '{{ current_user_roles(False)|tojson }}' as roles """, [], True, @@ -841,7 +844,9 @@ def test_none_operand_in_filter(login_as_admin, physical_dataset): @patch("superset.jinja_context.get_user_id", return_value=1) @patch("superset.jinja_context.get_username", return_value="abc") @patch("superset.jinja_context.get_user_email", return_value="abc@test.com") +@patch("superset.jinja_context.get_user_roles", return_value=["role1", "role2"]) def test_extra_cache_keys( + mock_get_user_roles, mock_user_email, mock_username, mock_user_id, @@ -883,7 +888,9 @@ def test_extra_cache_keys( @patch("superset.jinja_context.get_user_id", return_value=1) @patch("superset.jinja_context.get_username", return_value="abc") @patch("superset.jinja_context.get_user_email", return_value="abc@test.com") +@patch("superset.jinja_context.get_user_roles", return_value=["role1", "role2"]) def test_extra_cache_keys_in_sql_expression( + mock_get_user_roles, mock_user_email, mock_username, mock_user_id, diff --git a/tests/unit_tests/jinja_context_test.py b/tests/unit_tests/jinja_context_test.py index 5cc6f218a89..1654e267814 100644 --- a/tests/unit_tests/jinja_context_test.py +++ b/tests/unit_tests/jinja_context_test.py @@ -21,6 +21,7 @@ from datetime import datetime from typing import Any import pytest +from flask_appbuilder.security.sqla.models import Role from freezegun import freeze_time from jinja2 import DebugUndefined from jinja2.sandbox import SandboxedEnvironment @@ -360,6 +361,7 @@ def test_user_macros(mocker: MockerFixture): - ``current_user_id`` - ``current_username`` - ``current_user_email`` + - ``current_user_roles`` """ mock_g = mocker.patch("superset.utils.core.g") mock_cache_key_wrapper = mocker.patch( @@ -368,11 +370,13 @@ def test_user_macros(mocker: MockerFixture): mock_g.user.id = 1 mock_g.user.username = "my_username" mock_g.user.email = "my_email@test.com" + mock_g.user.roles = [Role(name="my_role1"), Role(name="my_role2")] cache = ExtraCache() assert cache.current_user_id() == 1 assert cache.current_username() == "my_username" assert cache.current_user_email() == "my_email@test.com" - assert mock_cache_key_wrapper.call_count == 3 + assert cache.current_user_roles() == ["my_role1", "my_role2"] + assert mock_cache_key_wrapper.call_count == 4 def test_user_macros_without_cache_key_inclusion(mocker: MockerFixture): @@ -386,10 +390,12 @@ def test_user_macros_without_cache_key_inclusion(mocker: MockerFixture): mock_g.user.id = 1 mock_g.user.username = "my_username" mock_g.user.email = "my_email@test.com" + mock_g.user.roles = [Role(name="my_role1"), Role(name="my_role2")] cache = ExtraCache() assert cache.current_user_id(False) == 1 assert cache.current_username(False) == "my_username" assert cache.current_user_email(False) == "my_email@test.com" + assert cache.current_user_roles(False) == ["my_role1", "my_role2"] assert mock_cache_key_wrapper.call_count == 0 @@ -403,6 +409,7 @@ def test_user_macros_without_user_info(mocker: MockerFixture): assert cache.current_user_id() == None # noqa: E711 assert cache.current_username() == None # noqa: E711 assert cache.current_user_email() == None # noqa: E711 + assert cache.current_user_roles() == None # noqa: E711 def test_where_in() -> None: From 6f69c84d10c3bf9f4bfa20ae876d1dfb7cc695e3 Mon Sep 17 00:00:00 2001 From: Chris <144701341+chrisvnimbus@users.noreply.github.com> Date: Mon, 24 Mar 2025 23:19:59 +0100 Subject: [PATCH 08/12] fix: key error in frontend on disallowed GSheets (#32792) --- superset/views/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superset/views/base.py b/superset/views/base.py index fe4ec0e0ab8..5b7ef99b026 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -319,7 +319,10 @@ def cached_common_bootstrap_data( # pylint: disable=unused-argument # verify client has google sheets installed available_specs = get_available_engine_specs() - frontend_config["HAS_GSHEETS_INSTALLED"] = bool(available_specs[GSheetsEngineSpec]) + frontend_config["HAS_GSHEETS_INSTALLED"] = ( + GSheetsEngineSpec in available_specs + and bool(available_specs[GSheetsEngineSpec]) + ) language = locale.language if locale else "en" From 7d77dc4fd2001bfa1a752fb9a2da0cdc7349f886 Mon Sep 17 00:00:00 2001 From: Fardin Mustaque <105560328+fardin-developer@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:52:15 +0530 Subject: [PATCH 09/12] fix: Time Comparison Feature Reverts Metric Labels to Metric Keys in Table Charts (#32665) Co-authored-by: Fardin Mustaque --- .../plugin-chart-table/src/TableChart.tsx | 8 +- .../plugin-chart-table/src/transformProps.ts | 5 ++ .../plugins/plugin-chart-table/src/types.ts | 2 + .../test/TableChart.test.tsx | 80 +++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-table/src/TableChart.tsx b/superset-frontend/plugins/plugin-chart-table/src/TableChart.tsx index 77905ea562e..95121a317d4 100644 --- a/superset-frontend/plugins/plugin-chart-table/src/TableChart.tsx +++ b/superset-frontend/plugins/plugin-chart-table/src/TableChart.tsx @@ -605,6 +605,8 @@ export default function TableChart( // Calculate the number of placeholder columns needed before the current header const startPosition = value[0]; const colSpan = value.length; + // Retrieve the originalLabel from the first column in this group + const originalLabel = columnsMeta[value[0]]?.originalLabel || key; // Add placeholder for columns before this header for (let i = currentColumnIndex; i < startPosition; i += 1) { @@ -620,7 +622,7 @@ export default function TableChart( // Add the current header headers.push( - {key} + {originalLabel} ( ), Footer: totals ? ( i === 0 ? ( - +
(
) : ( - + {formatColumnValue(column, totals[key])[1]} ) diff --git a/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts b/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts index 48871e4ea41..d62d9cb92c9 100644 --- a/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts @@ -347,6 +347,7 @@ const processComparisonColumns = ( } = props; const savedFormat = columnFormats?.[col.key]; const savedCurrency = currencyFormats?.[col.key]; + const originalLabel = col.label; if ( (col.isMetric || col.isPercentMetric) && !col.key.includes(comparisonSuffix) && @@ -355,6 +356,7 @@ const processComparisonColumns = ( return [ { ...col, + originalLabel, label: t('Main'), key: `${t('Main')} ${col.key}`, config: getComparisonColConfig(t('Main'), col.key, columnConfig), @@ -368,6 +370,7 @@ const processComparisonColumns = ( }, { ...col, + originalLabel, label: `#`, key: `# ${col.key}`, config: getComparisonColConfig(`#`, col.key, columnConfig), @@ -381,6 +384,7 @@ const processComparisonColumns = ( }, { ...col, + originalLabel, label: `△`, key: `△ ${col.key}`, config: getComparisonColConfig(`△`, col.key, columnConfig), @@ -394,6 +398,7 @@ const processComparisonColumns = ( }, { ...col, + originalLabel, label: `%`, key: `% ${col.key}`, config: getComparisonColConfig(`%`, col.key, columnConfig), diff --git a/superset-frontend/plugins/plugin-chart-table/src/types.ts b/superset-frontend/plugins/plugin-chart-table/src/types.ts index 1ec3cbe29d7..62a666a88e7 100644 --- a/superset-frontend/plugins/plugin-chart-table/src/types.ts +++ b/superset-frontend/plugins/plugin-chart-table/src/types.ts @@ -56,6 +56,8 @@ export interface DataColumnMeta { key: string; // `label` is verbose column name used for rendering label: string; + // `originalLabel` preserves the original label when time comparison transforms the labels + originalLabel?: string; dataType: GenericDataType; formatter?: | TimeFormatter diff --git a/superset-frontend/plugins/plugin-chart-table/test/TableChart.test.tsx b/superset-frontend/plugins/plugin-chart-table/test/TableChart.test.tsx index b21a657b815..b74e1ffccf4 100644 --- a/superset-frontend/plugins/plugin-chart-table/test/TableChart.test.tsx +++ b/superset-frontend/plugins/plugin-chart-table/test/TableChart.test.tsx @@ -175,6 +175,75 @@ describe('plugin-chart-table', () => { ?.formatter?.(0.123456); expect(formattedPercentMetric).toBe('0.123'); }); + + it('should set originalLabel for comparison columns when time_compare and comparison_type are set', () => { + const transformedProps = transformProps(testData.comparison); + + // Check if comparison columns are processed + const comparisonColumns = transformedProps.columns.filter( + col => + col.label === 'Main' || + col.label === '#' || + col.label === '△' || + col.label === '%', + ); + + expect(comparisonColumns.length).toBeGreaterThan(0); + expect(comparisonColumns.some(col => col.label === 'Main')).toBe(true); + expect(comparisonColumns.some(col => col.label === '#')).toBe(true); + expect(comparisonColumns.some(col => col.label === '△')).toBe(true); + expect(comparisonColumns.some(col => col.label === '%')).toBe(true); + + // Verify originalLabel for metric_1 comparison columns + const mainMetric1 = transformedProps.columns.find( + col => col.key === 'Main metric_1', + ); + expect(mainMetric1).toBeDefined(); + expect(mainMetric1?.originalLabel).toBe('metric_1'); + + const hashMetric1 = transformedProps.columns.find( + col => col.key === '# metric_1', + ); + expect(hashMetric1).toBeDefined(); + expect(hashMetric1?.originalLabel).toBe('metric_1'); + + const deltaMetric1 = transformedProps.columns.find( + col => col.key === '△ metric_1', + ); + expect(deltaMetric1).toBeDefined(); + expect(deltaMetric1?.originalLabel).toBe('metric_1'); + + const percentMetric1 = transformedProps.columns.find( + col => col.key === '% metric_1', + ); + expect(percentMetric1).toBeDefined(); + expect(percentMetric1?.originalLabel).toBe('metric_1'); + + // Verify originalLabel for metric_2 comparison columns + const mainMetric2 = transformedProps.columns.find( + col => col.key === 'Main metric_2', + ); + expect(mainMetric2).toBeDefined(); + expect(mainMetric2?.originalLabel).toBe('metric_2'); + + const hashMetric2 = transformedProps.columns.find( + col => col.key === '# metric_2', + ); + expect(hashMetric2).toBeDefined(); + expect(hashMetric2?.originalLabel).toBe('metric_2'); + + const deltaMetric2 = transformedProps.columns.find( + col => col.key === '△ metric_2', + ); + expect(deltaMetric2).toBeDefined(); + expect(deltaMetric2?.originalLabel).toBe('metric_2'); + + const percentMetric2 = transformedProps.columns.find( + col => col.key === '% metric_2', + ); + expect(percentMetric2).toBeDefined(); + expect(percentMetric2?.originalLabel).toBe('metric_2'); + }); }); describe('TableChart', () => { @@ -400,6 +469,17 @@ describe('plugin-chart-table', () => { ); expect(getComputedStyle(screen.getByText('N/A')).background).toBe(''); }); + it('should display originalLabel in grouped headers', () => { + render( + + + , + ); + + const groupHeaders = screen.getAllByRole('columnheader'); + expect(groupHeaders[0]).toHaveTextContent('metric_1'); + expect(groupHeaders[1]).toHaveTextContent('metric_2'); + }); }); it('render cell bars properly, and only when it is toggled on in both regular and percent metrics', () => { From 4a70065e5f6840fbcd657f50045c825a7b1d58d6 Mon Sep 17 00:00:00 2001 From: "JUST.in DO IT" Date: Tue, 25 Mar 2025 10:18:55 -0700 Subject: [PATCH 10/12] fix(log): store navigation path to get correct logging path (#32795) --- .../src/middleware/logger.test.js | 22 ++- .../src/middleware/loggerMiddleware.js | 142 ++++++++++-------- 2 files changed, 93 insertions(+), 71 deletions(-) diff --git a/superset-frontend/src/middleware/logger.test.js b/superset-frontend/src/middleware/logger.test.js index 629940b969e..0a28d42fdef 100644 --- a/superset-frontend/src/middleware/logger.test.js +++ b/superset-frontend/src/middleware/logger.test.js @@ -20,7 +20,10 @@ import sinon from 'sinon'; import { SupersetClient } from '@superset-ui/core'; import logger from 'src/middleware/loggerMiddleware'; import { LOG_EVENT } from 'src/logger/actions'; -import { LOG_ACTIONS_LOAD_CHART } from 'src/logger/LogUtils'; +import { + LOG_ACTIONS_LOAD_CHART, + LOG_ACTIONS_SPA_NAVIGATION, +} from 'src/logger/LogUtils'; describe('logger middleware', () => { const dashboardId = 123; @@ -40,7 +43,6 @@ describe('logger middleware', () => { eventData: { key: 'value', start_offset: 100, - path: `/dashboard/${dashboardId}/`, }, }, }; @@ -82,11 +84,19 @@ describe('logger middleware', () => { }); it('should include ts, start_offset, event_name, impression_id, source, and source_id in every event', () => { - logger(mockStore)(next)(action); + const fetchLog = logger(mockStore)(next); + fetchLog({ + type: LOG_EVENT, + payload: { + eventName: LOG_ACTIONS_SPA_NAVIGATION, + eventData: { path: `/dashboard/${dashboardId}/` }, + }, + }); timeSandbox.clock.tick(2000); - - expect(SupersetClient.post.callCount).toBe(1); - const { events } = SupersetClient.post.getCall(0).args[0].postPayload; + fetchLog(action); + timeSandbox.clock.tick(2000); + expect(SupersetClient.post.callCount).toBe(2); + const { events } = SupersetClient.post.getCall(1).args[0].postPayload; const mockEventdata = action.payload.eventData; const mockEventname = action.payload.eventName; expect(events[0]).toMatchObject({ diff --git a/superset-frontend/src/middleware/loggerMiddleware.js b/superset-frontend/src/middleware/loggerMiddleware.js index a3eb6295712..c008d027b3b 100644 --- a/superset-frontend/src/middleware/loggerMiddleware.js +++ b/superset-frontend/src/middleware/loggerMiddleware.js @@ -23,7 +23,10 @@ import { SupersetClient } from '@superset-ui/core'; import { safeStringify } from '../utils/safeStringify'; import { LOG_EVENT } from '../logger/actions'; -import { LOG_EVENT_TYPE_TIMING } from '../logger/LogUtils'; +import { + LOG_EVENT_TYPE_TIMING, + LOG_ACTIONS_SPA_NAVIGATION, +} from '../logger/LogUtils'; import DebouncedMessageQueue from '../utils/DebouncedMessageQueue'; const LOG_ENDPOINT = '/superset/log/?explode=events'; @@ -67,78 +70,87 @@ const logMessageQueue = new DebouncedMessageQueue({ delayThreshold: 1000, }); let lastEventId = 0; -const loggerMiddleware = store => next => action => { - if (action.type !== LOG_EVENT) { - return next(action); - } +const loggerMiddleware = store => next => { + let navPath; + return action => { + if (action.type !== LOG_EVENT) { + return next(action); + } - const { dashboardInfo, explore, impressionId, dashboardLayout, sqlLab } = - store.getState(); - let logMetadata = { - impression_id: impressionId, - version: 'v2', - }; - const { eventName } = action.payload; - let { eventData = {} } = action.payload; - - const path = eventData.path || window?.location?.href; - - if (dashboardInfo?.id && path?.includes('/dashboard/')) { - logMetadata = { - source: 'dashboard', - source_id: dashboardInfo.id, - dashboard_id: dashboardInfo.id, - ...logMetadata, + const { dashboardInfo, explore, impressionId, dashboardLayout, sqlLab } = + store.getState(); + let logMetadata = { + impression_id: impressionId, + version: 'v2', }; - } else if (explore?.slice) { - logMetadata = { - source: 'explore', - source_id: explore.slice ? explore.slice.slice_id : 0, - ...(explore.slice.slice_id && { slice_id: explore.slice.slice_id }), - ...logMetadata, - }; - } else if (path?.includes('/sqllab/')) { - const editor = sqlLab.queryEditors.find( - ({ id }) => id === sqlLab.tabHistory.slice(-1)[0], - ); - logMetadata = { - source: 'sqlLab', - source_id: editor?.id, - db_id: editor?.dbId, - schema: editor?.schema, - }; - } + const { eventName } = action.payload; + let { eventData = {} } = action.payload; + + if (eventName === LOG_ACTIONS_SPA_NAVIGATION) { + navPath = eventData.path; + } + const path = navPath || window?.location?.href; + + if (dashboardInfo?.id && path?.includes('/dashboard/')) { + logMetadata = { + source: 'dashboard', + source_id: dashboardInfo.id, + dashboard_id: dashboardInfo.id, + ...logMetadata, + }; + } else if (explore?.slice) { + logMetadata = { + source: 'explore', + source_id: explore.slice ? explore.slice.slice_id : 0, + ...(explore.slice.slice_id && { slice_id: explore.slice.slice_id }), + ...logMetadata, + }; + } else if (path?.includes('/sqllab/')) { + const editor = sqlLab.queryEditors.find( + ({ id }) => id === sqlLab.tabHistory.slice(-1)[0], + ); + logMetadata = { + source: 'sqlLab', + source_id: editor?.id, + db_id: editor?.dbId, + schema: editor?.schema, + }; + } - eventData = { - ...logMetadata, - ts: new Date().getTime(), - event_name: eventName, - ...eventData, - }; - if (LOG_EVENT_TYPE_TIMING.has(eventName)) { eventData = { + ...logMetadata, + ts: new Date().getTime(), + event_name: eventName, ...eventData, - event_type: 'timing', - trigger_event: lastEventId, }; - } else { - lastEventId = nanoid(); - eventData = { - ...eventData, - event_type: 'user', - event_id: lastEventId, - visibility: document.visibilityState, - }; - } + if (LOG_EVENT_TYPE_TIMING.has(eventName)) { + eventData = { + ...eventData, + event_type: 'timing', + trigger_event: lastEventId, + }; + } else { + lastEventId = nanoid(); + eventData = { + ...eventData, + event_type: 'user', + event_id: lastEventId, + visibility: document.visibilityState, + }; + } - if (eventData.target_id && dashboardLayout?.present?.[eventData.target_id]) { - const { meta } = dashboardLayout.present[eventData.target_id]; - // chart name or tab/header text - eventData.target_name = meta.chartId ? meta.sliceName : meta.text; - } + if ( + eventData.target_id && + dashboardLayout?.present?.[eventData.target_id] + ) { + const { meta } = dashboardLayout.present[eventData.target_id]; + // chart name or tab/header text + eventData.target_name = meta.chartId ? meta.sliceName : meta.text; + } - logMessageQueue.append(eventData); - return eventData; + logMessageQueue.append(eventData); + return eventData; + }; }; export default loggerMiddleware; From b5cb5f4525a4c78ef2fa2e81f443efcc21e43410 Mon Sep 17 00:00:00 2001 From: Joe Li Date: Tue, 25 Mar 2025 10:26:37 -0700 Subject: [PATCH 11/12] chore: updating files for release 4.1.2 (#32831) --- .github/ISSUE_TEMPLATE/bug-report.yml | 2 +- CHANGELOG/4.1.1.md | 50 ++++++++++++++++ CHANGELOG/4.1.2.md | 83 +++++++++++++++++++++++++++ UPDATING.md | 7 ++- 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 CHANGELOG/4.1.1.md create mode 100644 CHANGELOG/4.1.2.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index e6543af1ec7..124392ca875 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -41,7 +41,7 @@ body: label: Superset version options: - master / latest-dev - - "4.1.1" + - "4.1.2" - "4.0.2" validations: required: true diff --git a/CHANGELOG/4.1.1.md b/CHANGELOG/4.1.1.md new file mode 100644 index 00000000000..da51e78d12d --- /dev/null +++ b/CHANGELOG/4.1.1.md @@ -0,0 +1,50 @@ + + +## Change Log + +### 4.1 (Fri Nov 15 22:13:57 2024 +0530) + +**Database Migrations** + +**Features** + +**Fixes** + +- [#30886](https://github.com/apache/superset/pull/30886) fix: blocks UI elements on right side (@samarsrivastav) +- [#30859](https://github.com/apache/superset/pull/30859) fix(package.json): Pin luxon version to unblock master (@geido) +- [#30588](https://github.com/apache/superset/pull/30588) fix(explore): column data type tooltip format (@mistercrunch) +- [#29911](https://github.com/apache/superset/pull/29911) fix: Rename database from 'couchbasedb' to 'couchbase' in documentation and db_engine_specs (@ayush-couchbase) +- [#30828](https://github.com/apache/superset/pull/30828) fix(TimezoneSelector): Failing unit tests due to timezone change (@geido) +- [#30875](https://github.com/apache/superset/pull/30875) fix: don't show metadata for embedded dashboards (@sadpandajoe) +- [#30851](https://github.com/apache/superset/pull/30851) fix: Graph chart colors (@michael-s-molina) +- [#29867](https://github.com/apache/superset/pull/29867) fix(capitalization): Capitalizing a button. (@rusackas) +- [#29782](https://github.com/apache/superset/pull/29782) fix(translations): Translate embedded errors (@rusackas) +- [#29772](https://github.com/apache/superset/pull/29772) fix: Fixing incomplete string escaping. (@rusackas) +- [#29725](https://github.com/apache/superset/pull/29725) fix(frontend/docker, ci): fix borked Docker build due to Lerna v8 uplift (@hainenber) + +**Others** + +- [#30576](https://github.com/apache/superset/pull/30576) chore: add link to Superset when report error (@eschutho) +- [#29786](https://github.com/apache/superset/pull/29786) refactor(Slider): Upgrade Slider to Antd 5 (@geido) +- [#29674](https://github.com/apache/superset/pull/29674) refactor(ChartCreation): Migrate tests to RTL (@rtexelm) +- [#29843](https://github.com/apache/superset/pull/29843) refactor(controls): Migrate AdhocMetricOption.test to RTL (@rtexelm) +- [#29845](https://github.com/apache/superset/pull/29845) refactor(controls): Migrate MetricDefinitionValue.test to RTL (@rtexelm) +- [#28424](https://github.com/apache/superset/pull/28424) docs: Check markdown files for bad links using linkinator (@rusackas) +- [#29768](https://github.com/apache/superset/pull/29768) docs(contributing): fix broken link to translations sub-section (@sfirke) diff --git a/CHANGELOG/4.1.2.md b/CHANGELOG/4.1.2.md new file mode 100644 index 00000000000..917092b0518 --- /dev/null +++ b/CHANGELOG/4.1.2.md @@ -0,0 +1,83 @@ + + +## Change Log + +### 4.1.2 (Fri Mar 7 13:28:05 2025 -0800) + +**Database Migrations** + +- [#32538](https://github.com/apache/superset/pull/32538) fix(migrations): Handle comparator None in old time comparison migration (@Antonio-RiveroMartnez) +- [#32155](https://github.com/apache/superset/pull/32155) fix(migrations): Handle no params in time comparison migration (@Antonio-RiveroMartnez) +- [#31185](https://github.com/apache/superset/pull/31185) fix: check for column before adding in migrations (@betodealmeida) + +**Features** + +- [#29974](https://github.com/apache/superset/pull/29974) feat(sqllab): Adds refresh button to table metadata in SQL Lab (@Usiel) + +**Fixes** + +- [#32515](https://github.com/apache/superset/pull/32515) fix(sqllab): Allow clear on schema and catalog (@justinpark) +- [#32500](https://github.com/apache/superset/pull/32500) fix: dashboard, chart and dataset import validation (@dpgaspar) +- [#31353](https://github.com/apache/superset/pull/31353) fix(sqllab): duplicate error message (@betodealmeida) +- [#31407](https://github.com/apache/superset/pull/31407) fix: Big Number side cut fixed (@fardin-developer) +- [#31480](https://github.com/apache/superset/pull/31480) fix(sunburst): Use metric label from verbose map (@gerbermichi) +- [#31427](https://github.com/apache/superset/pull/31427) fix(tags): clean up bulk create api and schema (@villebro) +- [#31334](https://github.com/apache/superset/pull/31334) fix(docs): add custom editUrl path for intro page (@dwgrossberg) +- [#31353](https://github.com/apache/superset/pull/31353) fix(sqllab): duplicate error message (@betodealmeida) +- [#31323](https://github.com/apache/superset/pull/31323) fix: Use clickhouse sqlglot dialect for YDB (@vgvoleg) +- [#31198](https://github.com/apache/superset/pull/31198) fix: add more clickhouse disallowed functions on config (@dpgaspar) +- [#31194](https://github.com/apache/superset/pull/31194) fix(embedded): Hide anchor links in embedded mode (@Vitor-Avila) +- [#31960](https://github.com/apache/superset/pull/31960) fix(sqllab): Missing allowHTML props in ResultTableExtension (@justinpark) +- [#31332](https://github.com/apache/superset/pull/31332) fix: prevent multiple pvm errors on migration (@eschutho) +- [#31437](https://github.com/apache/superset/pull/31437) fix(database import): Gracefully handle error to get catalog schemas (@Vitor-Avila) +- [#31173](https://github.com/apache/superset/pull/31173) fix: cache-warmup fails (@nsivarajan) +- [#30442](https://github.com/apache/superset/pull/30442) fix(fe/src/dashboard): optional chaining for possibly nullable parent attribute in LayoutItem type (@hainenber) +- [#31639](https://github.com/apache/superset/pull/31639) fix(sqllab): unable to update saved queries (@DamianPendrak) +- [#29898](https://github.com/apache/superset/pull/29898) fix: parse pandas pivot null values (@eschutho) +- [#31414](https://github.com/apache/superset/pull/31414) fix(Pivot Table): Fix column width to respect currency config (@Vitor-Avila) +- [#31335](https://github.com/apache/superset/pull/31335) fix(histogram): axis margin padding consistent with other graphs (@tatiana-cherne) +- [#31301](https://github.com/apache/superset/pull/31301) fix(AllEntitiesTable): show Tags (@alexandrusoare) +- [#31329](https://github.com/apache/superset/pull/31329) fix: pass string to `process_template` (@betodealmeida) +- [#31341](https://github.com/apache/superset/pull/31341) fix(pinot): remove query aliases from SELECT and ORDER BY clauses in Pinot (@yuribogomolov) +- [#31308](https://github.com/apache/superset/pull/31308) fix: annotations on horizontal bar chart (@DamianPendrak) +- [#31294](https://github.com/apache/superset/pull/31294) fix(sqllab): Remove update_saved_query_exec_info to reduce lag (@justinpark) +- [#30897](https://github.com/apache/superset/pull/30897) fix: Exception handling for SQL Lab views (@michael-s-molina) +- [#31199](https://github.com/apache/superset/pull/31199) fix(Databricks): Escape catalog and schema names in pre-queries (@Vitor-Avila) +- [#31265](https://github.com/apache/superset/pull/31265) fix(trino): db session error in handle cursor (@justinpark) +- [#31024](https://github.com/apache/superset/pull/31024) fix(dataset): use sqlglot for DML check (@betodealmeida) +- [#29885](https://github.com/apache/superset/pull/29885) fix: add mutator to get_columns_description (@eschutho) +- [#30821](https://github.com/apache/superset/pull/30821) fix: x axis title disappears when editing bar chart (@DamianPendrak) +- [#31181](https://github.com/apache/superset/pull/31181) fix: Time-series Line Chart Display unnecessary total (@michael-s-molina) +- [#31163](https://github.com/apache/superset/pull/31163) fix(Dashboard): Backward compatible shared_label_colors field (@geido) +- [#31156](https://github.com/apache/superset/pull/31156) fix: check orderby (@betodealmeida) +- [#31154](https://github.com/apache/superset/pull/31154) fix: Remove unwanted commit on Trino's handle_cursor (@michael-s-molina) +- [#31151](https://github.com/apache/superset/pull/31151) fix: Revert "feat(trino): Add functionality to upload data (#29164)" (@michael-s-molina) +- [#31031](https://github.com/apache/superset/pull/31031) fix(Dashboard): Ensure shared label colors are updated (@geido) +- [#30967](https://github.com/apache/superset/pull/30967) fix(release validation): scripts now support RSA and EDDSA keys. (@rusackas) +- [#30881](https://github.com/apache/superset/pull/30881) fix(Dashboard): Native & Cross-Filters Scoping Performance (@geido) +- [#30887](https://github.com/apache/superset/pull/30887) fix(imports): import query_context for imports with charts (@lindenh) +- [#31008](https://github.com/apache/superset/pull/31008) fix(explore): verified props is not updated (@justinpark) +- [#30646](https://github.com/apache/superset/pull/30646) fix(Dashboard): Retain colors when color scheme not set (@geido) +- [#30962](https://github.com/apache/superset/pull/30962) fix(Dashboard): Exclude edit param in async screenshot (@geido) + +**Others** + +- [#32043](https://github.com/apache/superset/pull/32043) chore: Skip the creation of secondary perms during catalog migrations (@Vitor-Avila) +- [#30865](https://github.com/apache/superset/pull/30865) docs: Updating 4.1 Release Notes (@yousoph) diff --git a/UPDATING.md b/UPDATING.md index f0dbf1e5f0a..7fd97a4dab0 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -33,12 +33,10 @@ assists people when migrating to a new version. - [31794](https://github.com/apache/superset/pull/31794) Removed the previously deprecated `DASHBOARD_CROSS_FILTERS` feature flag - [31774](https://github.com/apache/superset/pull/31774): Fixes the spelling of the `USE-ANALAGOUS-COLORS` feature flag. Please update any scripts/configuration item to use the new/corrected `USE-ANALOGOUS-COLORS` flag spelling. - [31582](https://github.com/apache/superset/pull/31582) Removed the legacy Area, Bar, Event Flow, Heatmap, Histogram, Line, Sankey, and Sankey Loop charts. They were all automatically migrated to their ECharts counterparts with the exception of the Event Flow and Sankey Loop charts which were removed as they were not actively maintained and not widely used. If you were using the Event Flow or Sankey Loop charts, you will need to find an alternative solution. -- [31198](https://github.com/apache/superset/pull/31198) Disallows by default the use of the following ClickHouse functions: "version", "currentDatabase", "hostName". - [29798](https://github.com/apache/superset/pull/29798) Since 3.1.0, the intial schedule for an alert or report was mistakenly offset by the specified timezone's relation to UTC. The initial schedule should now begin at the correct time. - [30021](https://github.com/apache/superset/pull/30021) The `dev` layer in our Dockerfile no long includes firefox binaries, only Chromium to reduce bloat/docker-build-time. - [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`. - [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before. -- [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context. - [31413](https://github.com/apache/superset/pull/31413) Enable the DATE_FORMAT_IN_EMAIL_SUBJECT feature flag to allow users to specify a date format for the email subject, which will then be replaced with the actual date. - [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ... - [31503](https://github.com/apache/superset/pull/31503) Deprecating python 3.9.x support, 3.11 is now the recommended version and 3.10 is still supported over the Superset 5.0 lifecycle. @@ -51,6 +49,11 @@ assists people when migrating to a new version. ### Potential Downtime +## 4.1.2 + +- [31198](https://github.com/apache/superset/pull/31198) Disallows by default the use of the following ClickHouse functions: "version", "currentDatabase", "hostName". +- [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context. + ## 4.1.0 - [29274](https://github.com/apache/superset/pull/29274): We made it easier to trigger CI on your From b624919d2f9853d624c6f9ce19ea6617934b6297 Mon Sep 17 00:00:00 2001 From: "Michael S. Molina" <70410625+michael-s-molina@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:29:19 -0300 Subject: [PATCH 12/12] fix: Bump FAB to 4.6.1 (#32848) --- pyproject.toml | 2 +- requirements/base.txt | 2 +- requirements/development.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97c8252a419..2885418397a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "cryptography>=42.0.4, <45.0.0", "deprecation>=2.1.0, <2.2.0", "flask>=2.2.5, <3.0.0", - "flask-appbuilder>=4.6.0, <5.0.0", + "flask-appbuilder>=4.6.1, <5.0.0", "flask-caching>=2.1.0, <3", "flask-compress>=1.13, <2.0", "flask-talisman>=1.0.0, <2.0", diff --git a/requirements/base.txt b/requirements/base.txt index 3923bc2278e..8fc8fa2966b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -118,7 +118,7 @@ flask==2.3.3 # flask-session # flask-sqlalchemy # flask-wtf -flask-appbuilder==4.6.0 +flask-appbuilder==4.6.1 # via apache-superset (pyproject.toml) flask-babel==2.0.0 # via flask-appbuilder diff --git a/requirements/development.txt b/requirements/development.txt index 86be03515c8..8e563829a52 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -202,7 +202,7 @@ flask==2.3.3 # flask-sqlalchemy # flask-testing # flask-wtf -flask-appbuilder==4.6.0 +flask-appbuilder==4.6.1 # via # -c requirements/base.txt # apache-superset