fix: add DateOffset to json serializer (#32532)

This commit is contained in:
Elizabeth Thompson
2025-03-07 16:15:06 -08:00
committed by GitHub
parent 4c3aae7583
commit 33aa9030bf
4 changed files with 561 additions and 545 deletions

View File

@@ -15,13 +15,21 @@
# specific language governing permissions and limitations
# under the License.
import copy
import datetime
import math
import uuid
from datetime import date, datetime, time, timedelta
from decimal import Decimal
from unittest.mock import MagicMock
import numpy as np
import pandas as pd
import pytest
from superset.utils import json
from superset.utils.core import (
zlib_compress,
zlib_decompress,
)
def test_json_loads():
@@ -94,19 +102,35 @@ def test_json_dumps_encoding():
def test_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
"datetime": datetime(2021, 1, 1, 0, 0, 0),
"date": date(2021, 1, 1),
"dttm": datetime(2020, 1, 1),
"dt": date(2020, 1, 1),
"t": time(),
}
json_str = json.dumps(data, default=json.json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
assert reloaded_data["date"] == "2021-01-01"
assert reloaded_data["dttm"] == "2020-01-01T00:00:00"
assert reloaded_data["dt"] == "2020-01-01"
assert reloaded_data["t"] == "00:00:00"
assert json.json_iso_dttm_ser(np.int64(1)) == 1
assert (
json.json_iso_dttm_ser(np.datetime64(), pessimistic=True)
== "Unserializable [<class 'numpy.datetime64'>]"
)
with pytest.raises(TypeError):
json.json_iso_dttm_ser(np.datetime64())
def test_pessimistic_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
"datetime": datetime(2021, 1, 1, 0, 0, 0),
"date": date(2021, 1, 1),
}
json_str = json.dumps(data, default=json.pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
@@ -192,3 +216,57 @@ def test_sensitive_fields() -> None:
"user_token": "NEW_TOKEN",
},
}
def test_base_json_conv():
assert json.base_json_conv(np.bool_(1)) is True
assert json.base_json_conv(np.int64(1)) == 1
assert json.base_json_conv(np.array([1, 2, 3])) == [1, 2, 3]
assert json.base_json_conv(np.array(None)) is None
assert json.base_json_conv({1}) == [1]
assert json.base_json_conv(Decimal("1.0")) == 1.0
assert isinstance(json.base_json_conv(uuid.uuid4()), str)
assert json.base_json_conv(time(12, 0)) == "12:00:00"
assert json.base_json_conv(timedelta(0)) == "0:00:00"
assert json.base_json_conv(b"") == ""
assert isinstance(json.base_json_conv(b"\xff\xfe"), str)
assert json.base_json_conv(pd.DateOffset(days=1)) == "DateOffset(days=1)"
assert json.base_json_conv(b"\xff") == "[bytes]"
with pytest.raises(TypeError):
json.base_json_conv(np.datetime64())
def test_zlib_compression():
json_str = '{"test": 1}'
blob = zlib_compress(json_str)
got_str = zlib_decompress(blob)
assert json_str == got_str
def test_json_int_dttm_ser():
dttm = datetime(2020, 1, 1)
ts = 1577836800000.0
assert json.json_int_dttm_ser(dttm) == ts
assert json.json_int_dttm_ser(date(2020, 1, 1)) == ts
assert json.json_int_dttm_ser(datetime(1970, 1, 1)) == 0
assert json.json_int_dttm_ser(date(1970, 1, 1)) == 0
assert json.json_int_dttm_ser(dttm + timedelta(milliseconds=1)) == (ts + 1)
assert json.json_int_dttm_ser(np.int64(1)) == 1
with pytest.raises(TypeError):
json.json_int_dttm_ser(np.datetime64())
def test_format_timedelta():
assert json.format_timedelta(timedelta(0)) == "0:00:00"
assert json.format_timedelta(timedelta(days=1)) == "1 day, 0:00:00"
assert json.format_timedelta(timedelta(minutes=-6)) == "-0:06:00"
assert (
json.format_timedelta(timedelta(0) - timedelta(days=1, hours=5, minutes=6))
== "-1 day, 5:06:00"
)
assert (
json.format_timedelta(timedelta(0) - timedelta(days=16, hours=4, minutes=3))
== "-16 days, 4:03:00"
)

View File

@@ -21,6 +21,7 @@ from unittest.mock import MagicMock, patch
import pandas as pd
import pytest
from flask import current_app
from pytest_mock import MockerFixture
from superset.exceptions import SupersetException
@@ -32,10 +33,14 @@ from superset.utils.core import (
generic_find_fk_constraint_name,
get_datasource_full_name,
get_query_source_from_request,
get_stacktrace,
get_user_agent,
is_test,
merge_extra_filters,
merge_request_params,
normalize_dttm_col,
parse_boolean_string,
parse_js_uri_path_item,
QueryObjectFilterClause,
QuerySource,
remove_extra_adhoc_filters,
@@ -441,3 +446,472 @@ def test_get_user_agent(mocker: MockerFixture) -> None:
assert get_user_agent(database_mock, QuerySource.DASHBOARD) == "mydb DASHBOARD", (
"the custom user agent function result should have been returned"
)
def test_merge_extra_filters():
# does nothing if no extra filters
form_data = {"A": 1, "B": 2, "c": "test"}
expected = {**form_data, "adhoc_filters": [], "applied_time_extras": {}}
merge_extra_filters(form_data)
assert form_data == expected
# empty extra_filters
form_data = {"A": 1, "B": 2, "c": "test", "extra_filters": []}
expected = {
"A": 1,
"B": 2,
"c": "test",
"adhoc_filters": [],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
# copy over extra filters into empty filters
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": "someval"},
{"col": "B", "op": "==", "val": ["c1", "c2"]},
]
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
"isExtra": True,
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"filterOptionName": "6c178d069965f1c02640661280415d96",
"isExtra": True,
"operator": "==",
"subject": "B",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
# adds extra filters to existing filters
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": "someval"},
{"col": "B", "op": "==", "val": ["c1", "c2"]},
],
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["G1", "g2"],
"expressionType": "SIMPLE",
"operator": "!=",
"subject": "D",
}
],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["G1", "g2"],
"expressionType": "SIMPLE",
"operator": "!=",
"subject": "D",
},
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
"isExtra": True,
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"filterOptionName": "6c178d069965f1c02640661280415d96",
"isExtra": True,
"operator": "==",
"subject": "B",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
# adds extra filters to existing filters and sets time options
form_data = {
"extra_filters": [
{"col": "__time_range", "op": "in", "val": "1 year ago :"},
{"col": "__time_col", "op": "in", "val": "birth_year"},
{"col": "__time_grain", "op": "in", "val": "years"},
{"col": "A", "op": "like", "val": "hello"},
]
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "hello",
"expressionType": "SIMPLE",
"filterOptionName": "e3cbdd92a2ae23ca92c6d7fca42e36a6",
"isExtra": True,
"operator": "like",
"subject": "A",
}
],
"time_range": "1 year ago :",
"granularity_sqla": "birth_year",
"time_grain_sqla": "years",
"applied_time_extras": {
"__time_range": "1 year ago :",
"__time_col": "birth_year",
"__time_grain": "years",
},
}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_ignores_empty_filters():
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": ""},
{"col": "B", "op": "==", "val": []},
]
}
expected = {"adhoc_filters": [], "applied_time_extras": {}}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_ignores_nones():
form_data = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "",
"expressionType": "SIMPLE",
"operator": "in",
"subject": None,
}
],
"extra_filters": [{"col": "B", "op": "==", "val": []}],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "",
"expressionType": "SIMPLE",
"operator": "in",
"subject": None,
}
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_ignores_equal_filters():
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": "someval"},
{"col": "B", "op": "==", "val": ["c1", "c2"]},
{"col": "c", "op": "in", "val": ["c1", 1, None]},
],
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
{
"clause": "WHERE",
"comparator": ["c1", 1, None],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "c",
},
],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
{
"clause": "WHERE",
"comparator": ["c1", 1, None],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "c",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_merges_different_val_types():
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": ["g1", "g2"]},
{"col": "B", "op": "==", "val": ["c1", "c2"]},
],
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
{
"clause": "WHERE",
"comparator": ["g1", "g2"],
"expressionType": "SIMPLE",
"filterOptionName": "c11969c994b40a83a4ae7d48ff1ea28e",
"isExtra": True,
"operator": "in",
"subject": "a",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": "someval"},
{"col": "B", "op": "==", "val": ["c1", "c2"]},
],
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["g1", "g2"],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["g1", "g2"],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
{
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
"isExtra": True,
"operator": "in",
"subject": "a",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_adds_unequal_lists():
form_data = {
"extra_filters": [
{"col": "a", "op": "in", "val": ["g1", "g2", "g3"]},
{"col": "B", "op": "==", "val": ["c1", "c2", "c3"]},
],
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["g1", "g2"],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
],
}
expected = {
"adhoc_filters": [
{
"clause": "WHERE",
"comparator": ["g1", "g2"],
"expressionType": "SIMPLE",
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2"],
"expressionType": "SIMPLE",
"operator": "==",
"subject": "B",
},
{
"clause": "WHERE",
"comparator": ["g1", "g2", "g3"],
"expressionType": "SIMPLE",
"filterOptionName": "21cbb68af7b17e62b3b2f75e2190bfd7",
"isExtra": True,
"operator": "in",
"subject": "a",
},
{
"clause": "WHERE",
"comparator": ["c1", "c2", "c3"],
"expressionType": "SIMPLE",
"filterOptionName": "0a8dcb928f1f4bba97643c6e68d672f1",
"isExtra": True,
"operator": "==",
"subject": "B",
},
],
"applied_time_extras": {},
}
merge_extra_filters(form_data)
assert form_data == expected
def test_merge_extra_filters_when_applied_time_extras_predefined():
form_data = {"applied_time_extras": {"__time_range": "Last week"}}
merge_extra_filters(form_data)
assert form_data == {
"applied_time_extras": {"__time_range": "Last week"},
"adhoc_filters": [],
}
def test_merge_request_params_when_url_params_undefined():
form_data = {"since": "2000", "until": "now"}
url_params = {"form_data": form_data, "dashboard_ids": "(1,2,3,4,5)"}
merge_request_params(form_data, url_params)
assert "url_params" in form_data.keys()
assert "dashboard_ids" in form_data["url_params"]
assert "form_data" not in form_data.keys()
def test_merge_request_params_when_url_params_predefined():
form_data = {
"since": "2000",
"until": "now",
"url_params": {"abc": "123", "dashboard_ids": "(1,2,3)"},
}
url_params = {"form_data": form_data, "dashboard_ids": "(1,2,3,4,5)"}
merge_request_params(form_data, url_params)
assert "url_params" in form_data.keys()
assert "abc" in form_data["url_params"]
assert url_params["dashboard_ids"] == form_data["url_params"]["dashboard_ids"]
def test_parse_js_uri_path_items_eval_undefined():
assert parse_js_uri_path_item("undefined", eval_undefined=True) is None
assert parse_js_uri_path_item("null", eval_undefined=True) is None
assert "undefined" == parse_js_uri_path_item("undefined")
assert "null" == parse_js_uri_path_item("null")
def test_parse_js_uri_path_items_unquote():
assert "slashed/name" == parse_js_uri_path_item("slashed%2fname")
assert "slashed%2fname" == parse_js_uri_path_item("slashed%2fname", unquote=False)
def test_parse_js_uri_path_items_item_optional():
assert parse_js_uri_path_item(None) is None
assert parse_js_uri_path_item("item") is not None
def test_get_stacktrace():
current_app.config["SHOW_STACKTRACE"] = True
try:
raise Exception("NONONO!")
except Exception:
stacktrace = get_stacktrace()
assert "NONONO" in stacktrace
current_app.config["SHOW_STACKTRACE"] = False
try:
raise Exception("NONONO!")
except Exception:
stacktrace = get_stacktrace()
assert stacktrace is None