QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-23 01:55:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support mulitple temporal filters in AdhocFilter and move the Time Section away (#21767)

Browse Source
This commit is contained in:
Yongjie Zhao
2022-11-02 08:21:17 +08:00
committed by GitHub
parent 25be9ab4bc
commit a9b229dd1d
59 changed files with 1276 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
tests/integration_tests/query_context_tests.py
View File

@@ -1018,3 +1018,85 @@ def test_time_grain_and_time_offset_on_legacy_query(app_context, physical_datase
}
)
)
def test_time_offset_with_temporal_range_filter(app_context, physical_dataset):
qc = QueryContextFactory().create(
datasource={
"type": physical_dataset.type,
"id": physical_dataset.id,
},
queries=[
{
"columns": [
{
"label": "col6",
"sqlExpression": "col6",
"columnType": "BASE_AXIS",
"timeGrain": "P3M",
}
],
"metrics": [
{
"label": "SUM(col1)",
"expressionType": "SQL",
"sqlExpression": "SUM(col1)",
}
],
"time_offsets": ["3 month ago"],
"filters": [
{
"col": "col6",
"op": "TEMPORAL_RANGE",
"val": "2002-01 : 2003-01",
}
],
}
],
result_type=ChartDataResultType.FULL,
force=True,
)
query_payload = qc.get_df_payload(qc.queries[0])
df = query_payload["df"]
"""
col6 SUM(col1) SUM(col1)__3 month ago
0 2002-01-01 3 NaN
1 2002-04-01 12 3.0
2 2002-07-01 21 12.0
3 2002-10-01 9 21.0
"""
assert df["SUM(col1)"].to_list() == [3, 12, 21, 9]
# df["SUM(col1)__3 month ago"].dtype is object so have to convert to float first
assert df["SUM(col1)__3 month ago"].astype("float").astype("Int64").to_list() == [
pd.NA,
3,
12,
21,
]
sqls = query_payload["query"].split(";")
"""
SELECT DATE_TRUNC('quarter', col6) AS col6,
SUM(col1) AS "SUM(col1)"
FROM physical_dataset
WHERE col6 >= TO_TIMESTAMP('2002-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US')
AND col6 < TO_TIMESTAMP('2003-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US')
GROUP BY DATE_TRUNC('quarter', col6)
LIMIT 10000;
SELECT DATE_TRUNC('quarter', col6) AS col6,
SUM(col1) AS "SUM(col1)"
FROM physical_dataset
WHERE col6 >= TO_TIMESTAMP('2001-10-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US')
AND col6 < TO_TIMESTAMP('2002-10-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US')
GROUP BY DATE_TRUNC('quarter', col6)
LIMIT 10000;
"""
assert (
re.search(r"WHERE col6 >= .*2002-01-01", sqls[0])
and re.search(r"AND col6 < .*2003-01-01", sqls[0])
) is not None
assert (
re.search(r"WHERE col6 >= .*2001-10-01", sqls[1])
and re.search(r"AND col6 < .*2002-10-01", sqls[1])
) is not None

59
tests/integration_tests/sqla_models_tests.py
View File

@@ -23,7 +23,6 @@ import pytest
import numpy as np
import pandas as pd
import sqlalchemy as sa
from flask import Flask
from pytest_mock import MockFixture
from sqlalchemy.sql import text
@@ -41,7 +40,6 @@ from superset.utils.core import (
FilterOperator,
GenericDataType,
TemporalType,
backend,
)
from superset.utils.database import get_example_database
from tests.integration_tests.fixtures.birth_names_dashboard import (
@@ -71,6 +69,7 @@ VIRTUAL_TABLE_STRING_TYPES: Dict[str, Pattern[str]] = {
class FilterTestCase(NamedTuple):
column: str
operator: str
value: Union[float, int, List[Any], str]
expected: Union[str, List[str]]
@@ -271,19 +270,22 @@ class TestDatabaseModel(SupersetTestCase):
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
def test_where_operators(self):
filters: Tuple[FilterTestCase, ...] = (
FilterTestCase(FilterOperator.IS_NULL, "", "IS NULL"),
FilterTestCase(FilterOperator.IS_NOT_NULL, "", "IS NOT NULL"),
FilterTestCase("num", FilterOperator.IS_NULL, "", "IS NULL"),
FilterTestCase("num", FilterOperator.IS_NOT_NULL, "", "IS NOT NULL"),
# Some db backends translate true/false to 1/0
FilterTestCase(FilterOperator.IS_TRUE, "", ["IS 1", "IS true"]),
FilterTestCase(FilterOperator.IS_FALSE, "", ["IS 0", "IS false"]),
FilterTestCase(FilterOperator.GREATER_THAN, 0, "> 0"),
FilterTestCase(FilterOperator.GREATER_THAN_OR_EQUALS, 0, ">= 0"),
FilterTestCase(FilterOperator.LESS_THAN, 0, "< 0"),
FilterTestCase(FilterOperator.LESS_THAN_OR_EQUALS, 0, "<= 0"),
FilterTestCase(FilterOperator.EQUALS, 0, "= 0"),
FilterTestCase(FilterOperator.NOT_EQUALS, 0, "!= 0"),
FilterTestCase(FilterOperator.IN, ["1", "2"], "IN (1, 2)"),
FilterTestCase(FilterOperator.NOT_IN, ["1", "2"], "NOT IN (1, 2)"),
FilterTestCase("num", FilterOperator.IS_TRUE, "", ["IS 1", "IS true"]),
FilterTestCase("num", FilterOperator.IS_FALSE, "", ["IS 0", "IS false"]),
FilterTestCase("num", FilterOperator.GREATER_THAN, 0, "> 0"),
FilterTestCase("num", FilterOperator.GREATER_THAN_OR_EQUALS, 0, ">= 0"),
FilterTestCase("num", FilterOperator.LESS_THAN, 0, "< 0"),
FilterTestCase("num", FilterOperator.LESS_THAN_OR_EQUALS, 0, "<= 0"),
FilterTestCase("num", FilterOperator.EQUALS, 0, "= 0"),
FilterTestCase("num", FilterOperator.NOT_EQUALS, 0, "!= 0"),
FilterTestCase("num", FilterOperator.IN, ["1", "2"], "IN (1, 2)"),
FilterTestCase("num", FilterOperator.NOT_IN, ["1", "2"], "NOT IN (1, 2)"),
FilterTestCase(
"ds", FilterOperator.TEMPORAL_RANGE, "2020 : 2021", "2020-01-01"
),
)
table = self.get_table(name="birth_names")
for filter_ in filters:
@@ -295,7 +297,11 @@ class TestDatabaseModel(SupersetTestCase):
"metrics": ["count"],
"is_timeseries": False,
"filter": [
{"col": "num", "op": filter_.operator, "val": filter_.value}
{
"col": filter_.column,
"op": filter_.operator,
"val": filter_.value,
}
],
"extras": {},
}
@@ -835,3 +841,26 @@ def test__normalize_prequery_result_type(
assert str(normalized) == str(result)
else:
assert normalized == result
def test__temporal_range_operator_in_adhoc_filter(app_context, physical_dataset):
result = physical_dataset.query(
{
"columns": ["col1", "col2"],
"filter": [
{
"col": "col5",
"val": "2000-01-05 : 2000-01-06",
"op": FilterOperator.TEMPORAL_RANGE.value,
},
{
"col": "col6",
"val": "2002-05-11 : 2002-05-12",
"op": FilterOperator.TEMPORAL_RANGE.value,
},
],
"is_timeseries": False,
}
)
df = pd.DataFrame(index=[0], data={"col1": 4, "col2": "e"})
assert df.equals(result.df)

10
tests/integration_tests/utils_tests.py
View File

@@ -39,6 +39,7 @@ from sqlalchemy.exc import ArgumentError
import tests.integration_tests.test_app
from superset import app, db, security_manager
from superset.constants import NO_TIME_RANGE
from superset.exceptions import CertificateException, SupersetException
from superset.models.core import Database, Log
from superset.models.dashboard import Dashboard
@@ -62,7 +63,6 @@ from superset.utils.core import (
merge_extra_filters,
merge_extra_form_data,
merge_request_params,
NO_TIME_RANGE,
normalize_dttm_col,
parse_ssl_cert,
parse_js_uri_path_item,
@@ -1060,7 +1060,7 @@ class TestUtils(SupersetTestCase):
df: pd.DataFrame,
timestamp_format: Optional[str],
offset: int,
time_shift: Optional[timedelta],
time_shift: Optional[str],
) -> pd.DataFrame:
df = df.copy()
normalize_dttm_col(
@@ -1091,9 +1091,9 @@ class TestUtils(SupersetTestCase):
)
# test offset and timedelta
assert normalize_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][
0
] == pd.Timestamp(2021, 2, 15, 20, 30, 0, 0)
assert normalize_col(df, None, 1, "30 minutes")[DTTM_ALIAS][0] == pd.Timestamp(
2021, 2, 15, 20, 30, 0, 0
)
# test numeric epoch_s format
df = pd.DataFrame([{"__timestamp": ts.timestamp(), "a": 1}])

94
tests/unit_tests/common/test_time_range_utils.py Normal file
View File

@@ -0,0 +1,94 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from unittest import mock
import pytest
from superset.common.utils.time_range_utils import (
get_since_until_from_query_object,
get_since_until_from_time_range,
)
def test__get_since_until_from_time_range():
assert get_since_until_from_time_range(time_range="2001 : 2002") == (
datetime(2001, 1, 1),
datetime(2002, 1, 1),
)
assert get_since_until_from_time_range(
time_range="2001 : 2002", time_shift="8 hours ago"
) == (
datetime(2000, 12, 31, 16, 0, 0),
datetime(2001, 12, 31, 16, 0, 0),
)
with mock.patch(
"superset.utils.date_parser.EvalDateTruncFunc.eval",
return_value=datetime(2000, 1, 1, 0, 0, 0),
):
assert (
get_since_until_from_time_range(
time_range="Last year",
extras={
"relative_end": "2100",
},
)
)[1] == datetime(2100, 1, 1, 0, 0)
with mock.patch(
"superset.utils.date_parser.EvalDateTruncFunc.eval",
return_value=datetime(2000, 1, 1, 0, 0, 0),
):
assert (
get_since_until_from_time_range(
time_range="Next year",
extras={
"relative_start": "2000",
},
)
)[0] == datetime(2000, 1, 1, 0, 0)
@pytest.mark.query_object(
{
"time_range": "2001 : 2002",
"time_shift": "8 hours ago",
}
)
def test__since_until_from_time_range(dummy_query_object):
assert get_since_until_from_query_object(dummy_query_object) == (
datetime(2000, 12, 31, 16, 0, 0),
datetime(2001, 12, 31, 16, 0, 0),
)
@pytest.mark.query_object(
{
"filters": [{"col": "dttm", "op": "TEMPORAL_RANGE", "val": "2001 : 2002"}],
"columns": [
{
"columnType": "BASE_AXIS",
"label": "dttm",
"sqlExpression": "dttm",
}
],
}
)
def test__since_until_from_adhoc_filters(dummy_query_object):
assert get_since_until_from_query_object(dummy_query_object) == (
datetime(2001, 1, 1, 0, 0, 0),
datetime(2002, 1, 1, 0, 0, 0),
)

27
tests/unit_tests/conftest.py
View File

@@ -18,6 +18,7 @@
import importlib
import os
import unittest.mock
from typing import Any, Callable, Iterator
import pytest
@@ -29,6 +30,8 @@ from sqlalchemy.orm.session import Session
from superset import security_manager
from superset.app import SupersetApp
from superset.common.chart_data import ChartDataResultType
from superset.common.query_object_factory import QueryObjectFactory
from superset.extensions import appbuilder
from superset.initialization import SupersetAppInitializer
@@ -136,3 +139,27 @@ def full_api_access(mocker: MockFixture) -> Iterator[None]:
mocker.patch.object(security_manager, "can_access_all_databases", return_value=True)
yield
@pytest.fixture
def dummy_query_object(request, app_context):
query_obj_marker = request.node.get_closest_marker("query_object")
result_type_marker = request.node.get_closest_marker("result_type")
if query_obj_marker is None:
query_object = {}
else:
query_object = query_obj_marker.args[0]
if result_type_marker is None:
result_type = ChartDataResultType.FULL
else:
result_type = result_type_marker.args[0]
yield QueryObjectFactory(
app_configurations={
"ROW_LIMIT": 100,
},
_datasource_dao=unittest.mock.Mock(),
session_maker=unittest.mock.Mock(),
).create(parent_result_type=result_type, **query_object)