mirror of
https://github.com/apache/superset.git
synced 2026-04-19 08:04:53 +00:00
fix: add DateOffset to json serializer (#32532)
This commit is contained in:
committed by
GitHub
parent
4c3aae7583
commit
33aa9030bf
@@ -15,13 +15,21 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import copy
|
||||
import datetime
|
||||
import math
|
||||
import uuid
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from decimal import Decimal
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from superset.utils import json
|
||||
from superset.utils.core import (
|
||||
zlib_compress,
|
||||
zlib_decompress,
|
||||
)
|
||||
|
||||
|
||||
def test_json_loads():
|
||||
@@ -94,19 +102,35 @@ def test_json_dumps_encoding():
|
||||
|
||||
def test_json_iso_dttm_ser():
|
||||
data = {
|
||||
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
|
||||
"date": datetime.date(2021, 1, 1),
|
||||
"datetime": datetime(2021, 1, 1, 0, 0, 0),
|
||||
"date": date(2021, 1, 1),
|
||||
"dttm": datetime(2020, 1, 1),
|
||||
"dt": date(2020, 1, 1),
|
||||
"t": time(),
|
||||
}
|
||||
|
||||
json_str = json.dumps(data, default=json.json_iso_dttm_ser)
|
||||
reloaded_data = json.loads(json_str)
|
||||
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
|
||||
assert reloaded_data["date"] == "2021-01-01"
|
||||
assert reloaded_data["dttm"] == "2020-01-01T00:00:00"
|
||||
assert reloaded_data["dt"] == "2020-01-01"
|
||||
assert reloaded_data["t"] == "00:00:00"
|
||||
assert json.json_iso_dttm_ser(np.int64(1)) == 1
|
||||
|
||||
assert (
|
||||
json.json_iso_dttm_ser(np.datetime64(), pessimistic=True)
|
||||
== "Unserializable [<class 'numpy.datetime64'>]"
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
json.json_iso_dttm_ser(np.datetime64())
|
||||
|
||||
|
||||
def test_pessimistic_json_iso_dttm_ser():
|
||||
data = {
|
||||
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
|
||||
"date": datetime.date(2021, 1, 1),
|
||||
"datetime": datetime(2021, 1, 1, 0, 0, 0),
|
||||
"date": date(2021, 1, 1),
|
||||
}
|
||||
json_str = json.dumps(data, default=json.pessimistic_json_iso_dttm_ser)
|
||||
reloaded_data = json.loads(json_str)
|
||||
@@ -192,3 +216,57 @@ def test_sensitive_fields() -> None:
|
||||
"user_token": "NEW_TOKEN",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_base_json_conv():
|
||||
assert json.base_json_conv(np.bool_(1)) is True
|
||||
assert json.base_json_conv(np.int64(1)) == 1
|
||||
assert json.base_json_conv(np.array([1, 2, 3])) == [1, 2, 3]
|
||||
assert json.base_json_conv(np.array(None)) is None
|
||||
assert json.base_json_conv({1}) == [1]
|
||||
assert json.base_json_conv(Decimal("1.0")) == 1.0
|
||||
assert isinstance(json.base_json_conv(uuid.uuid4()), str)
|
||||
assert json.base_json_conv(time(12, 0)) == "12:00:00"
|
||||
assert json.base_json_conv(timedelta(0)) == "0:00:00"
|
||||
assert json.base_json_conv(b"") == ""
|
||||
assert isinstance(json.base_json_conv(b"\xff\xfe"), str)
|
||||
assert json.base_json_conv(pd.DateOffset(days=1)) == "DateOffset(days=1)"
|
||||
assert json.base_json_conv(b"\xff") == "[bytes]"
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
json.base_json_conv(np.datetime64())
|
||||
|
||||
|
||||
def test_zlib_compression():
|
||||
json_str = '{"test": 1}'
|
||||
blob = zlib_compress(json_str)
|
||||
got_str = zlib_decompress(blob)
|
||||
assert json_str == got_str
|
||||
|
||||
|
||||
def test_json_int_dttm_ser():
|
||||
dttm = datetime(2020, 1, 1)
|
||||
ts = 1577836800000.0
|
||||
assert json.json_int_dttm_ser(dttm) == ts
|
||||
assert json.json_int_dttm_ser(date(2020, 1, 1)) == ts
|
||||
assert json.json_int_dttm_ser(datetime(1970, 1, 1)) == 0
|
||||
assert json.json_int_dttm_ser(date(1970, 1, 1)) == 0
|
||||
assert json.json_int_dttm_ser(dttm + timedelta(milliseconds=1)) == (ts + 1)
|
||||
assert json.json_int_dttm_ser(np.int64(1)) == 1
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
json.json_int_dttm_ser(np.datetime64())
|
||||
|
||||
|
||||
def test_format_timedelta():
|
||||
assert json.format_timedelta(timedelta(0)) == "0:00:00"
|
||||
assert json.format_timedelta(timedelta(days=1)) == "1 day, 0:00:00"
|
||||
assert json.format_timedelta(timedelta(minutes=-6)) == "-0:06:00"
|
||||
assert (
|
||||
json.format_timedelta(timedelta(0) - timedelta(days=1, hours=5, minutes=6))
|
||||
== "-1 day, 5:06:00"
|
||||
)
|
||||
assert (
|
||||
json.format_timedelta(timedelta(0) - timedelta(days=16, hours=4, minutes=3))
|
||||
== "-16 days, 4:03:00"
|
||||
)
|
||||
|
||||
@@ -21,6 +21,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from flask import current_app
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from superset.exceptions import SupersetException
|
||||
@@ -32,10 +33,14 @@ from superset.utils.core import (
|
||||
generic_find_fk_constraint_name,
|
||||
get_datasource_full_name,
|
||||
get_query_source_from_request,
|
||||
get_stacktrace,
|
||||
get_user_agent,
|
||||
is_test,
|
||||
merge_extra_filters,
|
||||
merge_request_params,
|
||||
normalize_dttm_col,
|
||||
parse_boolean_string,
|
||||
parse_js_uri_path_item,
|
||||
QueryObjectFilterClause,
|
||||
QuerySource,
|
||||
remove_extra_adhoc_filters,
|
||||
@@ -441,3 +446,472 @@ def test_get_user_agent(mocker: MockerFixture) -> None:
|
||||
assert get_user_agent(database_mock, QuerySource.DASHBOARD) == "mydb DASHBOARD", (
|
||||
"the custom user agent function result should have been returned"
|
||||
)
|
||||
|
||||
|
||||
def test_merge_extra_filters():
|
||||
# does nothing if no extra filters
|
||||
form_data = {"A": 1, "B": 2, "c": "test"}
|
||||
expected = {**form_data, "adhoc_filters": [], "applied_time_extras": {}}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
# empty extra_filters
|
||||
form_data = {"A": 1, "B": 2, "c": "test", "extra_filters": []}
|
||||
expected = {
|
||||
"A": 1,
|
||||
"B": 2,
|
||||
"c": "test",
|
||||
"adhoc_filters": [],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
# copy over extra filters into empty filters
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": "someval"},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2"]},
|
||||
]
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "6c178d069965f1c02640661280415d96",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
# adds extra filters to existing filters
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": "someval"},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2"]},
|
||||
],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["G1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "!=",
|
||||
"subject": "D",
|
||||
}
|
||||
],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["G1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "!=",
|
||||
"subject": "D",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "6c178d069965f1c02640661280415d96",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
# adds extra filters to existing filters and sets time options
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "__time_range", "op": "in", "val": "1 year ago :"},
|
||||
{"col": "__time_col", "op": "in", "val": "birth_year"},
|
||||
{"col": "__time_grain", "op": "in", "val": "years"},
|
||||
{"col": "A", "op": "like", "val": "hello"},
|
||||
]
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "hello",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "e3cbdd92a2ae23ca92c6d7fca42e36a6",
|
||||
"isExtra": True,
|
||||
"operator": "like",
|
||||
"subject": "A",
|
||||
}
|
||||
],
|
||||
"time_range": "1 year ago :",
|
||||
"granularity_sqla": "birth_year",
|
||||
"time_grain_sqla": "years",
|
||||
"applied_time_extras": {
|
||||
"__time_range": "1 year ago :",
|
||||
"__time_col": "birth_year",
|
||||
"__time_grain": "years",
|
||||
},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_ignores_empty_filters():
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": ""},
|
||||
{"col": "B", "op": "==", "val": []},
|
||||
]
|
||||
}
|
||||
expected = {"adhoc_filters": [], "applied_time_extras": {}}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_ignores_nones():
|
||||
form_data = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": None,
|
||||
}
|
||||
],
|
||||
"extra_filters": [{"col": "B", "op": "==", "val": []}],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": None,
|
||||
}
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_ignores_equal_filters():
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": "someval"},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2"]},
|
||||
{"col": "c", "op": "in", "val": ["c1", 1, None]},
|
||||
],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", 1, None],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "c",
|
||||
},
|
||||
],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", 1, None],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "c",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_merges_different_val_types():
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": ["g1", "g2"]},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2"]},
|
||||
],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "c11969c994b40a83a4ae7d48ff1ea28e",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": "someval"},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2"]},
|
||||
],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_adds_unequal_lists():
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
{"col": "a", "op": "in", "val": ["g1", "g2", "g3"]},
|
||||
{"col": "B", "op": "==", "val": ["c1", "c2", "c3"]},
|
||||
],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2", "g3"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "21cbb68af7b17e62b3b2f75e2190bfd7",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2", "c3"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "0a8dcb928f1f4bba97643c6e68d672f1",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
],
|
||||
"applied_time_extras": {},
|
||||
}
|
||||
merge_extra_filters(form_data)
|
||||
assert form_data == expected
|
||||
|
||||
|
||||
def test_merge_extra_filters_when_applied_time_extras_predefined():
|
||||
form_data = {"applied_time_extras": {"__time_range": "Last week"}}
|
||||
merge_extra_filters(form_data)
|
||||
|
||||
assert form_data == {
|
||||
"applied_time_extras": {"__time_range": "Last week"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
|
||||
def test_merge_request_params_when_url_params_undefined():
|
||||
form_data = {"since": "2000", "until": "now"}
|
||||
url_params = {"form_data": form_data, "dashboard_ids": "(1,2,3,4,5)"}
|
||||
merge_request_params(form_data, url_params)
|
||||
assert "url_params" in form_data.keys()
|
||||
assert "dashboard_ids" in form_data["url_params"]
|
||||
assert "form_data" not in form_data.keys()
|
||||
|
||||
|
||||
def test_merge_request_params_when_url_params_predefined():
|
||||
form_data = {
|
||||
"since": "2000",
|
||||
"until": "now",
|
||||
"url_params": {"abc": "123", "dashboard_ids": "(1,2,3)"},
|
||||
}
|
||||
url_params = {"form_data": form_data, "dashboard_ids": "(1,2,3,4,5)"}
|
||||
merge_request_params(form_data, url_params)
|
||||
assert "url_params" in form_data.keys()
|
||||
assert "abc" in form_data["url_params"]
|
||||
assert url_params["dashboard_ids"] == form_data["url_params"]["dashboard_ids"]
|
||||
|
||||
|
||||
def test_parse_js_uri_path_items_eval_undefined():
|
||||
assert parse_js_uri_path_item("undefined", eval_undefined=True) is None
|
||||
assert parse_js_uri_path_item("null", eval_undefined=True) is None
|
||||
assert "undefined" == parse_js_uri_path_item("undefined")
|
||||
assert "null" == parse_js_uri_path_item("null")
|
||||
|
||||
|
||||
def test_parse_js_uri_path_items_unquote():
|
||||
assert "slashed/name" == parse_js_uri_path_item("slashed%2fname")
|
||||
assert "slashed%2fname" == parse_js_uri_path_item("slashed%2fname", unquote=False)
|
||||
|
||||
|
||||
def test_parse_js_uri_path_items_item_optional():
|
||||
assert parse_js_uri_path_item(None) is None
|
||||
assert parse_js_uri_path_item("item") is not None
|
||||
|
||||
|
||||
def test_get_stacktrace():
|
||||
current_app.config["SHOW_STACKTRACE"] = True
|
||||
try:
|
||||
raise Exception("NONONO!")
|
||||
except Exception:
|
||||
stacktrace = get_stacktrace()
|
||||
assert "NONONO" in stacktrace
|
||||
|
||||
current_app.config["SHOW_STACKTRACE"] = False
|
||||
try:
|
||||
raise Exception("NONONO!")
|
||||
except Exception:
|
||||
stacktrace = get_stacktrace()
|
||||
assert stacktrace is None
|
||||
|
||||
Reference in New Issue
Block a user