mirror of
https://github.com/apache/superset.git
synced 2026-07-25 16:12:39 +00:00
caf017bd0b
Co-authored-by: Mike Bridge <michael.bridge@ext.preset.io> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
424 lines
17 KiB
Python
424 lines
17 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
||
# or more contributor license agreements. See the NOTICE file
|
||
# distributed with this work for additional information
|
||
# regarding copyright ownership. The ASF licenses this file
|
||
# to you under the Apache License, Version 2.0 (the
|
||
# "License"); you may not use this file except in compliance
|
||
# with the License. You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing,
|
||
# software distributed under the License is distributed on an
|
||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
# KIND, either express or implied. See the License for the
|
||
# specific language governing permissions and limitations
|
||
# under the License.
|
||
"""T044 — Performance validation for entity version history.
|
||
|
||
Skipped by default. Run on demand:
|
||
|
||
SUPERSET_PERF_VALIDATION=1 pytest \
|
||
tests/integration_tests/versioning/perf_validation_tests.py -v -s
|
||
|
||
Measures the three success criteria defined in the spec:
|
||
|
||
* SC-002: version list endpoint responds in under 1 second
|
||
* SC-004: save path p95 overhead under 50 ms with Continuum tracking
|
||
on vs. off (FR-014)
|
||
|
||
(SC-003, the restore-endpoint benchmark, lands with the restore endpoint
|
||
in a later PR.)
|
||
|
||
The test prints a summary table suitable for pasting into the PR
|
||
description. It also asserts each target so regressions fail loudly
|
||
when the harness is re-run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import statistics
|
||
import time
|
||
from typing import Any
|
||
|
||
import pytest
|
||
import sqlalchemy as sa
|
||
from sqlalchemy_continuum import version_class, versioning_manager
|
||
|
||
from superset.extensions import db
|
||
from superset.models.slice import Slice
|
||
from tests.integration_tests.base_tests import SupersetTestCase
|
||
from tests.integration_tests.constants import ADMIN_USERNAME
|
||
from tests.integration_tests.fixtures.birth_names_dashboard import ( # noqa: F401
|
||
load_birth_names_dashboard_with_slices,
|
||
load_birth_names_data,
|
||
)
|
||
|
||
SKIP_REASON = "Performance validation is manual. Set SUPERSET_PERF_VALIDATION=1 to run."
|
||
|
||
# Thresholds from spec.md §Success Criteria.
|
||
LIST_ENDPOINT_MAX_MS = 1000 # SC-002
|
||
SAVE_OVERHEAD_P95_MAX_MS = 50 # SC-004
|
||
|
||
# Activity-view thresholds (sc-107283 §Success Criteria).
|
||
ACTIVITY_ENDPOINT_P95_MAX_MS = 1500 # SC-AV-001
|
||
|
||
|
||
def _save_chart_once(chart: Slice, suffix: str) -> None:
|
||
"""One ORM-level save path, mimicking what ChartDAO.update does."""
|
||
chart.slice_name = f"{chart.slice_name[:64]}_{suffix}"
|
||
db.session.commit()
|
||
|
||
|
||
def _timings_ms(seconds: list[float]) -> dict[str, float]:
|
||
ms = sorted(s * 1000.0 for s in seconds)
|
||
return {
|
||
"p50": statistics.median(ms),
|
||
"p95": ms[int(len(ms) * 0.95) - 1] if len(ms) >= 20 else max(ms),
|
||
"max": max(ms),
|
||
"n": len(ms),
|
||
}
|
||
|
||
|
||
@pytest.mark.skipif(
|
||
not os.environ.get("SUPERSET_PERF_VALIDATION"),
|
||
reason=SKIP_REASON,
|
||
)
|
||
class PerfValidationTests(SupersetTestCase):
|
||
"""Runs only when SUPERSET_PERF_VALIDATION=1 is set."""
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _load_data(self, load_birth_names_dashboard_with_slices: Any) -> None: # noqa: F811, PT004
|
||
pass
|
||
|
||
def _seed_chart_with_n_versions(self, n: int) -> Slice:
|
||
"""Save a chart N times to produce N version rows."""
|
||
chart = db.session.query(Slice).first()
|
||
assert chart is not None, "birth_names fixture should provide charts"
|
||
|
||
for i in range(n):
|
||
_save_chart_once(chart, f"v{i}")
|
||
db.session.commit()
|
||
return chart
|
||
|
||
def test_sc002_list_endpoint_under_1s(self) -> None:
|
||
"""SC-002: list endpoint responds in under 1 second."""
|
||
self.login(ADMIN_USERNAME)
|
||
|
||
# Generate enough versions to exercise the retention-capped state.
|
||
chart = self._seed_chart_with_n_versions(24)
|
||
chart_uuid = str(chart.uuid)
|
||
url = f"/api/v1/chart/{chart_uuid}/versions/"
|
||
|
||
# Warm up the endpoint once (JIT caching, mapper configuration, etc.)
|
||
self.client.get(url)
|
||
|
||
timings: list[float] = []
|
||
for _ in range(10):
|
||
t0 = time.perf_counter()
|
||
response = self.client.get(url)
|
||
timings.append(time.perf_counter() - t0)
|
||
assert response.status_code == 200
|
||
|
||
stats = _timings_ms(timings)
|
||
print(
|
||
f"\n[SC-002] GET /versions/ (24 versions) "
|
||
f"p50={stats['p50']:.1f}ms p95={stats['p95']:.1f}ms "
|
||
f"max={stats['max']:.1f}ms n={stats['n']}"
|
||
)
|
||
assert stats["p95"] < LIST_ENDPOINT_MAX_MS, (
|
||
f"SC-002 failed: list endpoint p95 {stats['p95']:.1f}ms "
|
||
f">= {LIST_ENDPOINT_MAX_MS}ms"
|
||
)
|
||
|
||
# SC-003 (restore endpoint under 3s) intentionally omitted here: the
|
||
# version-restore route (POST /<uuid>/versions/<version_uuid>/restore)
|
||
# ships in a later PR, so there is nothing to time at this point in the
|
||
# stack. Re-add this benchmark alongside the restore endpoint.
|
||
|
||
def test_sc004_save_overhead_under_50ms(self) -> None:
|
||
"""SC-004: save path p95 overhead under 50ms (FR-014).
|
||
|
||
Toggling Continuum on and off mid-process corrupts its internal
|
||
``units_of_work`` state and is not a reliable measurement. Instead
|
||
this test directly measures the wall-clock time spent inside the
|
||
four session-level listeners Continuum attaches to
|
||
``sa.orm.session.Session`` — ``before_flush``, ``after_flush``,
|
||
``after_commit``, ``after_rollback`` — plus Superset's own
|
||
baseline / snapshot / retention-prune listeners (attached to
|
||
``db.session``). The cumulative listener time per save is the
|
||
marginal overhead version capture adds over a save with
|
||
versioning removed entirely, because without these listeners
|
||
the ORM would not execute any of that code.
|
||
|
||
The approach:
|
||
1. Wrap each known listener with a timing proxy that adds its
|
||
wall-clock time to a per-save accumulator.
|
||
2. Save the same chart N times, recording each save's
|
||
accumulator value.
|
||
3. Compute p50 / p95 of the per-save overhead.
|
||
|
||
The pass/fail gate uses full save latency, which includes both
|
||
Continuum's class-level listeners and Superset's listeners attached
|
||
to ``db.session``. The wrapped Continuum listeners remain a useful
|
||
diagnostic component without letting unmeasured listeners create a
|
||
false pass.
|
||
"""
|
||
self.login(ADMIN_USERNAME)
|
||
|
||
chart = db.session.query(Slice).first()
|
||
assert chart is not None
|
||
|
||
# Per-save accumulator incremented by the wrapped listeners.
|
||
acc: list[float] = [0.0]
|
||
|
||
def wrap_listener(original: Any) -> Any:
|
||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||
t0 = time.perf_counter()
|
||
try:
|
||
return original(*args, **kwargs)
|
||
finally:
|
||
acc[0] += time.perf_counter() - t0
|
||
|
||
wrapper.__wrapped__ = original # type: ignore[attr-defined]
|
||
return wrapper
|
||
|
||
# Instrument Continuum's four session listeners by detaching the
|
||
# bound method, wrapping, and re-attaching under a single-use
|
||
# listener handle we can cleanly remove on teardown.
|
||
session_target = sa.orm.session.Session
|
||
attached: list[tuple[str, Any]] = []
|
||
for event_name, listener in list(versioning_manager.session_listeners.items()):
|
||
sa.event.remove(session_target, event_name, listener)
|
||
wrapped = wrap_listener(listener)
|
||
sa.event.listen(session_target, event_name, wrapped)
|
||
attached.append((event_name, wrapped))
|
||
|
||
iterations: int = 100
|
||
warmup: int = 5
|
||
try:
|
||
# Warmup (first baseline INSERT, JIT, cache warming).
|
||
for i in range(warmup):
|
||
_save_chart_once(chart, f"warm_{i}")
|
||
acc[0] = 0.0
|
||
|
||
total_timings: list[float] = []
|
||
overhead_timings: list[float] = []
|
||
for i in range(iterations):
|
||
acc[0] = 0.0
|
||
t0 = time.perf_counter()
|
||
_save_chart_once(chart, f"run_{i}")
|
||
total_timings.append(time.perf_counter() - t0)
|
||
overhead_timings.append(acc[0])
|
||
finally:
|
||
for event_name, wrapped in attached:
|
||
sa.event.remove(session_target, event_name, wrapped)
|
||
sa.event.listen(
|
||
session_target,
|
||
event_name,
|
||
wrapped.__wrapped__,
|
||
)
|
||
|
||
total = _timings_ms(total_timings)
|
||
overhead = _timings_ms(overhead_timings)
|
||
|
||
ver_cls = version_class(Slice)
|
||
produced = db.session.query(ver_cls).filter(ver_cls.id == chart.id).count()
|
||
print(
|
||
f"\n[SC-004] save iterations={iterations} chart_id={chart.id} "
|
||
f"version_rows_produced={produced}"
|
||
)
|
||
print(
|
||
f"[SC-004] full save: "
|
||
f"p50={total['p50']:.2f}ms p95={total['p95']:.2f}ms "
|
||
f"max={total['max']:.2f}ms"
|
||
)
|
||
print(
|
||
f"[SC-004] version-cap overhead: "
|
||
f"p50={overhead['p50']:.2f}ms p95={overhead['p95']:.2f}ms "
|
||
f"max={overhead['max']:.2f}ms"
|
||
)
|
||
|
||
assert total["p95"] < SAVE_OVERHEAD_P95_MAX_MS, (
|
||
f"SC-004 failed: full save p95 "
|
||
f"{total['p95']:.2f}ms >= {SAVE_OVERHEAD_P95_MAX_MS}ms"
|
||
)
|
||
|
||
# ---- T045: Activity-view perf validation -----------------------------
|
||
|
||
def _seed_activity_history(self) -> str:
|
||
"""Generate dense history on the birth_names dashboard so the
|
||
activity endpoint has something realistic to read.
|
||
|
||
T045's spec target is "25 charts × 3 dataset windows each". The
|
||
birth_names fixture has ~12 charts on a single dataset (no
|
||
multi-dataset support without a bespoke fixture). We approximate
|
||
the load by: (a) editing many charts on the dashboard, (b)
|
||
editing the dataset's description several times, (c) editing the
|
||
dashboard's own title once. That yields ~30+ change records
|
||
spanning all three entity kinds — enough to exercise the
|
||
decoration, visibility, and impact-batch paths without needing a
|
||
multi-dataset fixture builder. Returns the dashboard UUID.
|
||
|
||
**Why this commits without rollback** (unlike the test bodies in
|
||
``activity_view_tests.py``): the whole point of a perf seed is
|
||
that the rows it produces have actually been persisted, so the
|
||
endpoint hit that follows reads a realistic state of the
|
||
``version_changes`` / shadow tables. T053's
|
||
``try/finally``+``rollback`` convention is for tests that
|
||
assert on *which records were captured*; here the seed IS the
|
||
setup, not the unit under test. The fixture's session-scoped
|
||
``_cleanup`` removes the dashboard / slices at session teardown,
|
||
which is when the shadow rows age out too.
|
||
"""
|
||
# pylint: disable=import-outside-toplevel
|
||
from superset.connectors.sqla.models import SqlaTable
|
||
from superset.models.dashboard import Dashboard
|
||
|
||
dashboard = (
|
||
db.session.query(Dashboard)
|
||
.filter(Dashboard.dashboard_title.like("USA Births%"))
|
||
.first()
|
||
)
|
||
dataset: SqlaTable | None = (
|
||
db.session.query(SqlaTable)
|
||
.filter(SqlaTable.table_name == "birth_names")
|
||
.first()
|
||
)
|
||
assert dashboard is not None
|
||
assert dataset is not None
|
||
dashboard_uuid = str(dashboard.uuid)
|
||
|
||
# Many chart edits — most of the activity volume.
|
||
for chart in dashboard.slices[:12]:
|
||
chart.slice_name = f"{chart.slice_name[:48]}_perf"
|
||
db.session.commit()
|
||
|
||
# A handful of dataset edits — exercises the impact-batch path
|
||
# (Dashboard path + SqlaTable related).
|
||
for i in range(5):
|
||
dataset.description = f"perf seed iteration {i}"
|
||
db.session.commit()
|
||
|
||
# One dashboard self-edit.
|
||
dashboard.dashboard_title = f"{dashboard.dashboard_title}_perf"
|
||
db.session.commit()
|
||
|
||
return dashboard_uuid
|
||
|
||
def test_av_sc001_activity_endpoint_p95_under_1500ms(self) -> None:
|
||
"""SC-AV-001: dashboard activity endpoint p95 < 1500ms across 50
|
||
invocations against a realistic history."""
|
||
self.login(ADMIN_USERNAME)
|
||
|
||
dashboard_uuid = self._seed_activity_history()
|
||
url = f"/api/v1/dashboard/{dashboard_uuid}/activity/"
|
||
|
||
# Warmup — JIT, mapper config, identity-map population.
|
||
for _ in range(3):
|
||
self.client.get(url)
|
||
|
||
timings: list[float] = []
|
||
for _ in range(50):
|
||
t0 = time.perf_counter()
|
||
response = self.client.get(url)
|
||
timings.append(time.perf_counter() - t0)
|
||
assert response.status_code == 200
|
||
|
||
stats = _timings_ms(timings)
|
||
body = response.get_json()
|
||
print(
|
||
f"\n[AV-SC-001] GET /dashboard/<uuid>/activity/ "
|
||
f"records_returned={len(body['result'])} count={body['count']}"
|
||
)
|
||
print(
|
||
f"[AV-SC-001] p50={stats['p50']:.1f}ms "
|
||
f"p95={stats['p95']:.1f}ms max={stats['max']:.1f}ms "
|
||
f"n={stats['n']}"
|
||
)
|
||
assert stats["p95"] < ACTIVITY_ENDPOINT_P95_MAX_MS, (
|
||
f"AV-SC-001 failed: activity endpoint p95 {stats['p95']:.1f}ms "
|
||
f">= {ACTIVITY_ENDPOINT_P95_MAX_MS}ms — profile the query "
|
||
f"plan and consider the T046 index migration (see "
|
||
f"specs/sc-107283/data-model.md §Possible additive indexes)"
|
||
)
|
||
|
||
def test_av_sc003_save_path_p95_unaffected_by_activity_view(self) -> None:
|
||
"""AV-SC-003: the activity-view feature is read-only. Save path
|
||
p95 must remain within sc-103156's SC-004 budget (50ms version-
|
||
capture overhead) even with the activity tables in place.
|
||
|
||
We re-measure the same overhead SC-004 measures, with the
|
||
activity-view branch's code in scope, to catch any accidental
|
||
regression from a save-path coupling.
|
||
"""
|
||
self.login(ADMIN_USERNAME)
|
||
# Seed some history so the M2M shadow + version_changes have
|
||
# enough rows that any pathological save-time read against them
|
||
# would surface.
|
||
self._seed_activity_history()
|
||
|
||
chart = db.session.query(Slice).first()
|
||
assert chart is not None
|
||
|
||
acc: list[float] = [0.0]
|
||
|
||
def wrap_listener(original: Any) -> Any:
|
||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||
t0 = time.perf_counter()
|
||
try:
|
||
return original(*args, **kwargs)
|
||
finally:
|
||
acc[0] += time.perf_counter() - t0
|
||
|
||
wrapper.__wrapped__ = original # type: ignore[attr-defined]
|
||
return wrapper
|
||
|
||
session_target = sa.orm.session.Session
|
||
attached: list[tuple[str, Any]] = []
|
||
for event_name, listener in list(versioning_manager.session_listeners.items()):
|
||
sa.event.remove(session_target, event_name, listener)
|
||
wrapped = wrap_listener(listener)
|
||
sa.event.listen(session_target, event_name, wrapped)
|
||
attached.append((event_name, wrapped))
|
||
|
||
iterations: int = 50
|
||
warmup: int = 3
|
||
try:
|
||
for i in range(warmup):
|
||
_save_chart_once(chart, f"av_warm_{i}")
|
||
acc[0] = 0.0
|
||
|
||
total_timings: list[float] = []
|
||
overhead_timings: list[float] = []
|
||
for i in range(iterations):
|
||
acc[0] = 0.0
|
||
t0: float = time.perf_counter()
|
||
_save_chart_once(chart, f"av_run_{i}")
|
||
total_timings.append(time.perf_counter() - t0)
|
||
overhead_timings.append(acc[0])
|
||
finally:
|
||
for event_name, wrapped in attached:
|
||
sa.event.remove(session_target, event_name, wrapped)
|
||
sa.event.listen(
|
||
session_target,
|
||
event_name,
|
||
wrapped.__wrapped__,
|
||
)
|
||
|
||
total: dict[str, float] = _timings_ms(total_timings)
|
||
overhead: dict[str, float] = _timings_ms(overhead_timings)
|
||
print(
|
||
f"\n[AV-SC-003] full save with activity-view in scope: "
|
||
f"p50={total['p50']:.2f}ms p95={total['p95']:.2f}ms "
|
||
f"max={total['max']:.2f}ms; "
|
||
f"Continuum component p95={overhead['p95']:.2f}ms"
|
||
)
|
||
assert total["p95"] < SAVE_OVERHEAD_P95_MAX_MS, (
|
||
f"AV-SC-003 failed: full save p95 "
|
||
f"{total['p95']:.2f}ms >= {SAVE_OVERHEAD_P95_MAX_MS}ms — "
|
||
f"the activity-view branch has regressed sc-103156's SC-004 "
|
||
f"budget; check for a new save-path read coupling."
|
||
)
|