Compare commits

...

3 Commits

Author SHA1 Message Date
Beto Dealmeida
a78e691c6e Address comments 2026-07-07 09:23:34 -04:00
Beto Dealmeida
47e03e8a93 Increase test coverage 2026-07-07 09:23:34 -04:00
Beto Dealmeida
93d3c876ae fix(semantic layers): improve time grain 2026-07-07 09:23:33 -04:00
5 changed files with 632 additions and 38 deletions

View File

@@ -67,7 +67,11 @@ const getMemoizedSectionsToRender = memoizeOne(
} = sections;
// list of datasource-specific controls that should be removed if the datasource is a specific type
const filterControlsForTypes = [DatasourceType.Query, DatasourceType.Table];
const filterControlsForTypes = [
DatasourceType.Query,
DatasourceType.Table,
DatasourceType.SemanticView,
];
const invalidControls = filterControlsForTypes.includes(datasourceType)
? ['granularity']
: ['granularity_sqla', 'time_grain_sqla'];

View File

@@ -337,18 +337,43 @@ def map_query_object(query_object: ValidatedQueryObject) -> list[SemanticQuery]:
metrics = [all_metrics[metric] for metric in (query_object.metrics or [])]
grain = _convert_time_grain(query_object.extras.get("time_grain_sqla"))
dimensions = [
dimension
for dimension in semantic_view.dimensions
if dimension.name in normalized_columns
and (
# if a grain is specified, only include the time dimension if its grain
# matches the requested grain
grain is None
or dimension.name != query_object.granularity
or dimension.grain == grain
time_axis_column = _get_time_axis_column(query_object, all_dimensions)
# A semantic view can expose multiple Dimension variants per name (one per
# supported time grain). Pick exactly one variant per selected column:
# for the time-axis column we honor the user's grain selection, falling
# back to the raw / no-grain variant when no exact match exists and then
# to any available variant so the axis is never silently dropped; for
# every other selected column we prefer the raw variant and otherwise
# take any available variant.
dimensions: list[Dimension] = []
seen_non_axis: dict[str, Dimension] = {}
axis_variants: list[Dimension] = []
axis_match: Dimension | None = None
for dimension in semantic_view.dimensions:
if dimension.name not in normalized_columns:
continue
if dimension.name == time_axis_column:
axis_variants.append(dimension)
if axis_match is None and dimension.grain == grain:
axis_match = dimension
continue
existing = seen_non_axis.get(dimension.name)
if existing is None or (existing.grain is not None and dimension.grain is None):
seen_non_axis[dimension.name] = dimension
if axis_match is not None:
dimensions.append(axis_match)
elif axis_variants:
# No variant matches the requested grain. Prefer the raw (grain=None)
# variant; otherwise pick a deterministic fallback so the axis stays
# on the query instead of being silently dropped.
raw_variant = next((v for v in axis_variants if v.grain is None), None)
dimensions.append(
raw_variant
if raw_variant is not None
else min(axis_variants, key=lambda v: v.grain.name if v.grain else "")
)
]
dimensions.extend(seen_non_axis.values())
order = _get_order_from_query_object(query_object, all_metrics, all_dimensions)
limit = query_object.row_limit
@@ -932,6 +957,50 @@ def _get_group_limit_filters(
return filters if filters else None
def _get_time_axis_column(
query_object: ValidatedQueryObject,
all_dimensions: dict[str, Dimension],
) -> str | None:
"""
Determine which selected column is the time-axis (the one a time grain
applies to).
Legacy time-series charts encode this as ``query_object.granularity``;
modern x-axis charts leave that empty and put the temporal column in
``query_object.columns`` instead, with the grain on
``extras["time_grain_sqla"]``. In that case we only claim an axis when
the selected columns include exactly one temporal dimension — otherwise
which one is the x-axis is ambiguous from the ``QueryObject`` alone
(form_data's ``x_axis`` is not available here). Returning ``None`` on
ambiguity lets the grain-application code fall back to raw variants for
every column rather than silently applying the grain to whichever
temporal column happens to be iterated first.
"""
if query_object.granularity:
return query_object.granularity
dimension_names = set(all_dimensions.keys())
temporal_columns: list[str] = []
for column in query_object.columns or []:
try:
name = _normalize_column(column, dimension_names)
except ValueError:
continue
dim = all_dimensions.get(name)
if dim is None:
continue
if (
pa.types.is_timestamp(dim.type)
or pa.types.is_date(dim.type)
or pa.types.is_time(dim.type)
):
temporal_columns.append(name)
if len(temporal_columns) == 1:
return temporal_columns[0]
return None
def _convert_time_grain(time_grain: str | None) -> Grain | None:
"""
Convert a time grain string (ISO 8601 duration) to a Grain instance.
@@ -1018,15 +1087,20 @@ def _validate_granularity(query_object: ValidatedQueryObject) -> None:
Make sure time column and time grain are valid.
"""
semantic_view = query_object.datasource.implementation
dimension_names = {dimension.name for dimension in semantic_view.dimensions}
all_dimensions = {
dimension.name: dimension for dimension in semantic_view.dimensions
}
dimension_names = set(all_dimensions.keys())
if time_column := query_object.granularity:
if time_column not in dimension_names:
raise ValueError(
"The time column must be defined in the Semantic View dimensions."
)
if (legacy_time_column := query_object.granularity) and (
legacy_time_column not in dimension_names
):
raise ValueError(
"The time column must be defined in the Semantic View dimensions."
)
if time_grain := query_object.extras.get("time_grain_sqla"):
time_column = _get_time_axis_column(query_object, all_dimensions)
if not time_column:
raise ValueError(
"A time column must be specified when a time grain is provided."

View File

@@ -323,8 +323,27 @@ class SemanticView(AuditMixinNullable, Model):
for metric in self.implementation.get_metrics()
]
@property
def _unique_dimensions(self) -> list[Any]:
# A semantic view may expose multiple ``Dimension`` objects sharing the
# same ``name`` but different grains (one variant per supported time
# grain). For column-list purposes we collapse these into a single
# entry; the available grains are surfaced separately via
# ``get_time_grains`` and ``data["time_grain_sqla"]``.
seen: dict[str, Any] = {}
for dimension in self.implementation.get_dimensions():
seen.setdefault(dimension.name, dimension)
return list(seen.values())
@property
def columns(self) -> list[ColumnMetadata]:
# ``expression`` is intentionally left unset: the explore UI uses a
# non-empty ``expression`` to mean "this column is a SQL-style adhoc
# expression" and renders it with the fx icon and no time-grain
# affordance. Semantic-view dimensions are physical from the UI's
# perspective; the backend is responsible for any underlying
# expression. ``dimension.definition`` is not surfaced to the UI —
# only ``dimension.description`` is passed through.
return [
ColumnMetadata(
column_name=dimension.name,
@@ -333,17 +352,17 @@ class SemanticView(AuditMixinNullable, Model):
or pa.types.is_time(dimension.type)
or pa.types.is_timestamp(dimension.type),
description=dimension.description,
expression=dimension.definition,
expression=None,
extra=json.dumps(
{"grain": dimension.grain.name if dimension.grain else None}
),
)
for dimension in self.implementation.get_dimensions()
for dimension in self._unique_dimensions
]
@property
def column_names(self) -> list[str]:
return [dimension.name for dimension in self.implementation.get_dimensions()]
return [dimension.name for dimension in self._unique_dimensions]
@property
def data(self) -> ExplorableData:
@@ -360,7 +379,9 @@ class SemanticView(AuditMixinNullable, Model):
"certified_by": None,
"column_name": dimension.name,
"description": dimension.description,
"expression": dimension.definition,
# See ``columns`` property: leaving ``expression`` empty
# avoids the fx-icon / adhoc treatment in the explore UI.
"expression": None,
"filterable": True,
"groupby": True,
"id": None,
@@ -375,7 +396,7 @@ class SemanticView(AuditMixinNullable, Model):
"verbose_name": None,
"warning_markdown": None,
}
for dimension in self.implementation.get_dimensions()
for dimension in self._unique_dimensions
],
"metrics": [
{
@@ -407,12 +428,9 @@ class SemanticView(AuditMixinNullable, Model):
"description": self.description,
"table_name": self.name,
"column_types": [
get_column_type(dimension.type)
for dimension in self.implementation.get_dimensions()
],
"column_names": [
dimension.name for dimension in self.implementation.get_dimensions()
get_column_type(dimension.type) for dimension in self._unique_dimensions
],
"column_names": [dimension.name for dimension in self._unique_dimensions],
# rare
"column_formats": {},
"datasource_name": self.name,
@@ -424,7 +442,12 @@ class SemanticView(AuditMixinNullable, Model):
"schema": None,
"catalog": None,
"main_dttm_col": None,
"time_grain_sqla": [],
# ``time_grain_sqla`` in ``ExplorableData`` is the ``(duration,
# name)`` tuple shape the explore UI consumes; the dict shape
# lives on ``get_time_grains``.
"time_grain_sqla": [
(grain["duration"], grain["name"]) for grain in self.get_time_grains()
],
"granularity_sqla": [],
"fetch_values_predicate": None,
"template_params": None,
@@ -470,15 +493,22 @@ class SemanticView(AuditMixinNullable, Model):
return 0
def get_time_grains(self) -> list[TimeGrainDict]:
return [
{
"name": dimension.grain.name,
"function": "",
"duration": dimension.grain.representation,
}
for dimension in self.implementation.get_dimensions()
if dimension.grain
]
# Return the union of grains across all time dimensions, deduped by
# ISO duration so the explore-time-grain dropdown shows each grain
# once even when multiple time columns expose the same set.
seen: dict[str, TimeGrainDict] = {}
for dimension in self.implementation.get_dimensions():
if not dimension.grain:
continue
seen.setdefault(
dimension.grain.representation,
{
"name": dimension.grain.name,
"function": "",
"duration": dimension.grain.representation,
},
)
return list(seen.values())
def has_drill_by_columns(self, column_names: list[str]) -> bool:
dimension_names = {

View File

@@ -51,6 +51,7 @@ from superset.semantic_layers.mapper import (
_get_group_limit_filters,
_get_group_limit_from_query_object,
_get_order_from_query_object,
_get_time_axis_column,
_get_time_bounds,
_get_time_filter,
_normalize_column,
@@ -1004,6 +1005,379 @@ def test_map_query_object_with_time_offsets(mock_datasource: MagicMock) -> None:
}
def _make_grain_variant_datasource(
mocker: MockerFixture,
granularity_dim_grain: Grain | None,
extra_dim_grain: Grain | None = None,
) -> MagicMock:
"""Datasource with raw + Hour + Day variants on ``order_date``."""
datasource = mocker.Mock()
base = {
"id": "orders.order_date",
"name": "order_date",
"type": pa.timestamp("us"),
"description": "Order date",
"definition": "order_date",
}
date_variants = {
Dimension(**base, grain=None),
Dimension(**base, grain=Grains.HOUR),
Dimension(**base, grain=Grains.DAY),
}
category = Dimension(
id="products.category",
name="category",
type=pa.utf8(),
description="Product category",
definition="category",
)
sales = Metric(
id="orders.total_sales",
name="total_sales",
type=pa.float64(),
definition="SUM(amount)",
description="Total sales",
)
implementation = MockSemanticView(
dimensions=date_variants | {category},
metrics={sales},
features=frozenset(),
)
datasource.implementation = implementation
datasource.fetch_values_predicate = None
return datasource
def test_map_query_object_picks_grain_variant_matching_user_selection(
mocker: MockerFixture,
) -> None:
"""Only the variant matching the user's grain is sent through."""
datasource = _make_grain_variant_datasource(
mocker, granularity_dim_grain=Grains.DAY
)
query_object = ValidatedQueryObject(
datasource=datasource,
metrics=["total_sales"],
columns=["category", "order_date"],
granularity="order_date",
extras={"time_grain_sqla": "P1D"},
)
result = map_query_object(query_object)
selected_grains = {dim.grain for dim in result[0].dimensions}
assert selected_grains == {
Grains.DAY,
None,
} # day for order_date, None for category
def test_map_query_object_picks_raw_variant_when_no_grain_selected(
mocker: MockerFixture,
) -> None:
"""
No grain selected — the time-axis column must collapse to the raw (grain=None)
variant rather than passing all grain variants through to the semantic view.
"""
datasource = _make_grain_variant_datasource(mocker, granularity_dim_grain=None)
query_object = ValidatedQueryObject(
datasource=datasource,
metrics=["total_sales"],
columns=["category", "order_date"],
granularity="order_date",
# No time_grain_sqla in extras
)
result = map_query_object(query_object)
order_date_dims = [dim for dim in result[0].dimensions if dim.name == "order_date"]
assert len(order_date_dims) == 1
assert order_date_dims[0].grain is None
def test_map_query_object_falls_back_when_no_grain_variant_matches(
mocker: MockerFixture,
) -> None:
"""
Regression: when the time-axis column exposes only grained variants (no
``grain=None``) and the user picks a grain that isn't in the list, the
old code silently dropped the axis. It must now fall back to a variant
so the axis stays on the query.
"""
datasource = mocker.Mock()
base = {
"id": "orders.order_date",
"name": "order_date",
"type": pa.timestamp("us"),
"description": "Order date",
"definition": "order_date",
}
# HOUR and DAY only — no grain=None, no MONTH.
variants = {
Dimension(**base, grain=Grains.HOUR),
Dimension(**base, grain=Grains.DAY),
}
sales = Metric(
id="orders.total_sales",
name="total_sales",
type=pa.float64(),
definition="SUM(amount)",
description="Total sales",
)
implementation = MockSemanticView(
dimensions=variants,
metrics={sales},
features=frozenset(),
)
datasource.implementation = implementation
datasource.fetch_values_predicate = None
query_object = ValidatedQueryObject(
datasource=datasource,
metrics=["total_sales"],
columns=["order_date"],
granularity="order_date",
extras={"time_grain_sqla": "P1M"}, # MONTH — not in variants
)
result = map_query_object(query_object)
order_date_dims = [d for d in result[0].dimensions if d.name == "order_date"]
assert len(order_date_dims) == 1
# Deterministic fallback: alphabetically first grain name — "Day" < "Hour".
assert order_date_dims[0].grain == Grains.DAY
def test_map_query_object_falls_back_to_raw_when_no_grain_variant_matches(
mocker: MockerFixture,
) -> None:
"""
When no grained variant matches the requested grain but a ``grain=None``
variant exists, the raw variant is preferred over any other grained
fallback.
"""
datasource = mocker.Mock()
base = {
"id": "orders.order_date",
"name": "order_date",
"type": pa.timestamp("us"),
"description": "Order date",
"definition": "order_date",
}
variants = {
Dimension(**base, grain=None),
Dimension(**base, grain=Grains.HOUR),
}
sales = Metric(
id="orders.total_sales",
name="total_sales",
type=pa.float64(),
definition="SUM(amount)",
description="Total sales",
)
implementation = MockSemanticView(
dimensions=variants,
metrics={sales},
features=frozenset(),
)
datasource.implementation = implementation
datasource.fetch_values_predicate = None
query_object = ValidatedQueryObject(
datasource=datasource,
metrics=["total_sales"],
columns=["order_date"],
granularity="order_date",
extras={"time_grain_sqla": "P1D"}, # DAY — not in variants
)
result = map_query_object(query_object)
order_date_dims = [d for d in result[0].dimensions if d.name == "order_date"]
assert len(order_date_dims) == 1
assert order_date_dims[0].grain is None
def test_map_query_object_picks_raw_variant_for_non_axis_time_dim(
mocker: MockerFixture,
) -> None:
"""
Multiple grain variants of a *non-axis* time dimension collapse to the raw
(grain=None) variant. ``order_date`` is the granularity axis here so
``shipped_at`` falls through to the non-axis branch with several variants
competing — exercising both arms of the ``existing is None or ...`` guard
in ``map_query_object``.
"""
datasource = mocker.Mock()
shipped_base = {
"id": "shipments.shipped_at",
"name": "shipped_at",
"type": pa.timestamp("us"),
"description": "Ship time",
"definition": "shipped_at",
}
shipped_variants = {
Dimension(**shipped_base, grain=None),
Dimension(**shipped_base, grain=Grains.HOUR),
Dimension(**shipped_base, grain=Grains.DAY),
}
order_date = Dimension(
id="orders.order_date",
name="order_date",
type=pa.timestamp("us"),
description="Order date",
definition="order_date",
grain=None,
)
sales = Metric(
id="orders.total_sales",
name="total_sales",
type=pa.float64(),
definition="SUM(amount)",
description="Total sales",
)
implementation = MockSemanticView(
dimensions=shipped_variants | {order_date},
metrics={sales},
features=frozenset(),
)
datasource.implementation = implementation
datasource.fetch_values_predicate = None
query_object = ValidatedQueryObject(
datasource=datasource,
metrics=["total_sales"],
columns=["order_date", "shipped_at"],
granularity="order_date",
)
result = map_query_object(query_object)
shipped_dims = [d for d in result[0].dimensions if d.name == "shipped_at"]
assert len(shipped_dims) == 1
assert shipped_dims[0].grain is None
def test_get_time_axis_column_returns_granularity_when_set(
mocker: MockerFixture,
) -> None:
"""Legacy path: ``granularity`` short-circuits scanning ``columns``."""
qo = mocker.Mock()
qo.granularity = "order_date"
assert _get_time_axis_column(qo, {}) == "order_date"
def test_get_time_axis_column_finds_temporal_in_columns(
mocker: MockerFixture,
) -> None:
"""Modern x_axis path: pick the first temporal dim from ``columns``."""
all_dims = {
"category": Dimension(
id="products.category",
name="category",
type=pa.utf8(),
definition="category",
),
"order_date": Dimension(
id="orders.order_date",
name="order_date",
type=pa.timestamp("us"),
definition="order_date",
),
}
qo = mocker.Mock()
qo.granularity = None
qo.columns = ["category", "order_date"]
assert _get_time_axis_column(qo, all_dims) == "order_date"
def test_get_time_axis_column_skips_unknown_columns(
mocker: MockerFixture,
) -> None:
"""A column not present in the semantic view is silently skipped."""
all_dims = {
"category": Dimension(
id="products.category",
name="category",
type=pa.utf8(),
definition="category",
),
}
qo = mocker.Mock()
qo.granularity = None
qo.columns = ["does_not_exist", "category"]
# No temporal dim in columns — returns None.
assert _get_time_axis_column(qo, all_dims) is None
def test_get_time_axis_column_skips_unparseable_adhoc_columns(
mocker: MockerFixture,
) -> None:
"""An adhoc column that ``_normalize_column`` rejects is silently skipped."""
all_dims = {
"order_date": Dimension(
id="orders.order_date",
name="order_date",
type=pa.timestamp("us"),
definition="order_date",
),
}
qo = mocker.Mock()
qo.granularity = None
# First column is an adhoc dict without ``isColumnReference`` — raises in
# ``_normalize_column`` and the loop should keep going.
qo.columns = [{"label": "unsupported", "sqlExpression": "lower(x)"}, "order_date"]
assert _get_time_axis_column(qo, all_dims) == "order_date"
def test_get_time_axis_column_returns_none_on_multiple_temporal_columns(
mocker: MockerFixture,
) -> None:
"""
Ambiguity guard: with ``granularity`` unset and more than one temporal
column selected, ``QueryObject`` alone cannot identify the x-axis
(``form_data`` is not carried through). We return ``None`` rather than
picking one arbitrarily, and the grain-application path falls back to
raw variants for every column.
"""
all_dims = {
"created_at": Dimension(
id="orders.created_at",
name="created_at",
type=pa.timestamp("us"),
definition="created_at",
),
"shipped_at": Dimension(
id="orders.shipped_at",
name="shipped_at",
type=pa.timestamp("us"),
definition="shipped_at",
),
}
qo = mocker.Mock()
qo.granularity = None
qo.columns = ["created_at", "shipped_at"]
assert _get_time_axis_column(qo, all_dims) is None
def test_get_time_axis_column_returns_none_when_no_temporal_columns(
mocker: MockerFixture,
) -> None:
"""Without ``granularity`` and without a temporal column we return None."""
all_dims = {
"category": Dimension(
id="products.category",
name="category",
type=pa.utf8(),
definition="category",
),
}
qo = mocker.Mock()
qo.granularity = None
qo.columns = ["category"]
assert _get_time_axis_column(qo, all_dims) is None
def test_convert_query_object_filter_unknown_operator(
mock_datasource: MagicMock,
) -> None:

View File

@@ -655,6 +655,118 @@ def test_semantic_view_data(
assert data["offset"] == 0
@pytest.fixture
def mock_grain_variant_dimensions() -> list[Dimension]:
"""Time column exposed as multiple Dimension variants, one per grain."""
base = {
"id": "orders.created_at",
"name": "created_at",
"type": pa.timestamp("us"),
"definition": "orders.created_at",
"description": "Order timestamp",
}
return [
Dimension(**base, grain=Grains.HOUR),
Dimension(**base, grain=Grains.DAY),
Dimension(**base, grain=Grains.MONTH),
Dimension(
id="products.category",
name="category",
type=pa.utf8(),
definition="products.category",
description="Product category",
grain=None,
),
]
def test_semantic_view_columns_dedupes_grain_variants(
mock_grain_variant_dimensions: list[Dimension],
) -> None:
"""Multiple grain variants of the same time column collapse to one column."""
impl = MagicMock()
impl.get_dimensions.return_value = mock_grain_variant_dimensions
view = SemanticView()
with patch.object(
SemanticView,
"implementation",
new_callable=lambda: property(lambda s: impl),
):
columns = view.columns
assert [c.column_name for c in columns] == ["created_at", "category"]
assert columns[0].is_dttm is True
assert view.column_names == ["created_at", "category"]
def test_semantic_view_get_time_grains_dedupes_across_dimensions(
mock_grain_variant_dimensions: list[Dimension],
) -> None:
"""Grains shared across multiple time dimensions are returned once each."""
extra_dim = Dimension(
id="shipments.shipped_at",
name="shipped_at",
type=pa.timestamp("us"),
definition="shipments.shipped_at",
description=None,
grain=Grains.DAY,
)
impl = MagicMock()
impl.get_dimensions.return_value = mock_grain_variant_dimensions + [extra_dim]
view = SemanticView()
with patch.object(
SemanticView,
"implementation",
new_callable=lambda: property(lambda s: impl),
):
grains = view.get_time_grains()
durations = sorted(grain["duration"] or "" for grain in grains)
assert durations == sorted(["PT1H", "P1D", "P1M"])
def test_semantic_view_data_populates_time_grain_sqla(
mock_grain_variant_dimensions: list[Dimension],
mock_metrics: list[Metric],
) -> None:
"""``data['time_grain_sqla']`` mirrors ``get_time_grains`` for the explore UI."""
from superset.semantic_layers.models import SemanticLayer
impl = MagicMock()
impl.get_dimensions.return_value = mock_grain_variant_dimensions
impl.get_metrics.return_value = mock_metrics
impl.uid.return_value = "semantic_view_uid_123"
layer = SemanticLayer()
layer.name = "My Semantic Layer"
layer.uuid = uuid.UUID("87654321-4321-8765-4321-876543218765")
layer.perm = "[My Semantic Layer](id:87654321432187654321876543218765)"
view = SemanticView()
view.name = "Orders View"
view.description = "View of order data"
view.id = 1
view.uuid = uuid.UUID("12345678-1234-5678-1234-567812345678")
view.semantic_layer_uuid = uuid.UUID("87654321-4321-8765-4321-876543218765")
view.semantic_layer = layer
view.cache_timeout = 3600
with patch.object(
SemanticView,
"implementation",
new_callable=lambda: property(lambda s: impl),
):
data = view.data
assert data["column_names"] == ["created_at", "category"]
assert len(data["columns"]) == 2
assert data["columns"][0]["is_dttm"] is True
# ``time_grain_sqla`` in ExplorableData is ``(duration, name)`` tuples.
grain_durations = sorted(entry[0] for entry in data["time_grain_sqla"])
assert grain_durations == sorted(["PT1H", "P1D", "P1M"])
def test_semantic_view_get_query_result(
mock_implementation: MagicMock,
) -> None: