superset2/superset/versioning/diff.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""Diff engine for the ``version_changes`` table (FR-016..FR-019).

Hand-rolled because:

- The on-disk ``path`` shape (array of segments) is a direct
  representation of our chosen format; external diff libraries
  return string paths or JSON-Pointer forms that would need
  translation.
- Kind classification (``filter`` vs ``metric`` vs ``field`` etc.)
  is co-located with diff walking, avoiding a second classification
  pass over the generic diff output.
- Child-collection identity uses natural keys (``column_name``,
  ``metric_name``, slice ``uuid``) — the same identity model
  ``DatasetDAO.update_columns`` settled on (ADR-004). External
  libraries default to list-index matching, which is wrong for our
  data.

See ADR (plan.md §"Key Design Decision: Hand-rolled diff engine") for
the full rationale.

All functions in this module are pure: they take dicts (or lists of
dicts) and return a list of :class:`ChangeRecord`. The ORM->dict
conversion and Continuum transaction lookup happen in the capture
listener (T048), not here. This keeps the engine unit-testable without
an app context or DB.
"""

from __future__ import annotations

from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Callable, Optional

from superset.utils import json as _json

# Columns that are always excluded from change records, regardless of
# what ``__versioned__`` says. ``id`` / ``uuid`` are stable identifiers
# (not edited in normal flows). The four audit fields change on every
# save — emitting records for them would double every history entry
# with meaningless "timestamp changed, user stamped" rows that the UI
# would have to filter out anyway.
_AUDIT_FIELDS: frozenset[str] = frozenset(
    {
        "id",
        "uuid",
        "created_on",
        "changed_on",
        "created_by_fk",
        "changed_by_fk",
    }
)

# Fields stripped from child-collection dict items (TableColumn,
# SqlMetric) before comparison and emission. ``changed_on`` /
# ``created_on`` / ``*_by_fk`` are audit fields that update on every
# save of the parent — without this filter, saving a dataset to add
# one column produces a record per existing column too (because their
# ``changed_on`` timestamps all refreshed). ``id`` and ``table_id``
# are implementation details — ``id`` can change under the
# ``override_columns`` delete-and-reinsert pattern (ADR-004) even
# when the column is semantically unchanged; ``table_id`` is the
# parent FK and never meaningfully differs within one dataset's
# history. ``uuid`` stays stable across normal saves and is kept so
# the renderer can use it for identity if it needs to.
_CHILD_ITEM_OPAQUE_FIELDS: frozenset[str] = frozenset(
    {
        "id",
        "table_id",
        "changed_on",
        "created_on",
        "changed_by_fk",
        "created_by_fk",
    }
)


def _strip_opaque_fields(item: Any) -> Any:
    """Return *item* with child-item audit/implementation fields removed.

    Pass-through for non-dict values (scalars, strings) — the strip
    only applies where it matters (dataset column / metric dicts).
    """
    if not isinstance(item, dict):
        return item
    return {k: v for k, v in item.items() if k not in _CHILD_ITEM_OPAQUE_FIELDS}


# Chart ``params`` sub-keys that are promoted to first-class kinds.
# Every other params sub-key falls through to ``kind="field"``.
_CHART_PARAMS_KIND_BY_KEY: dict[str, str] = {
    "adhoc_filters": "filter",
    "time_range": "time_range",
    "color_scheme": "color_palette",
    "metrics": "metric",
    "groupby": "dimension",
    "columns": "dimension",
}

# Chart ``params`` sub-keys that are machine-stamped on save and don't
# carry user-authored signal — same category as ``last_saved_at`` on
# the scalar side. ``slice_id`` is a self-reference to the chart's
# own primary id; Superset's save paths add or refresh it on every
# save, producing a spurious "field" record on the first save after
# a chart's params were stored without it.
_CHART_PARAMS_AUDIT_KEYS: frozenset[str] = frozenset({"slice_id"})


def scalar_fields_for(
    model_cls: Any,
    *,
    special: frozenset[str] = frozenset(),
    audit: frozenset[str] = frozenset(),
) -> frozenset[str]:
    """Scalar columns on ``model_cls`` that should produce change records.

    Derived from the model itself at call time so contributors (and
    downstream derivatives) don't have to maintain a parallel whitelist
    in this module. Adding a new column to ``Dashboard``, ``Slice``, or
    ``SqlaTable`` — whether upstream or in a fork — automatically flows
    through to ``version_changes`` on the next save.

    Excludes, in order:

    1. The model's own ``__versioned__.exclude`` list, so change records
       stay consistent with Continuum's shadow tables. If Continuum
       isn't tracking a column, the change log shouldn't either.
    2. :data:`_AUDIT_FIELDS` — ``id``, ``uuid``, and the audit
       timestamps / user-id columns shared across the three entity types.
    3. The caller's ``audit`` set — model-specific save-side-effect
       columns that aren't user-authored content. ``Slice.last_saved_at``
       / ``last_saved_by_fk`` are stamped on every chart save by
       ``UpdateChartCommand``, similar to how ``changed_on`` is stamped
       by the ORM event listener; emitting "field" records for them
       would noise up the change log with one entry per save that
       carries no user-meaningful signal.
    4. The caller's ``special`` set — columns handled by a dedicated
       differ elsewhere. ``Slice.params``, for example, is walked by
       :func:`diff_slice_params` to produce first-class ``filter`` /
       ``time_range`` / ``metric`` / ``dimension`` records; emitting
       it as a single opaque ``field`` would defeat that.
    """
    try:
        table = model_cls.__table__
    except AttributeError:
        return frozenset()
    columns = frozenset(c.name for c in table.columns)
    continuum_exclude = frozenset(
        getattr(model_cls, "__versioned__", {}).get("exclude", []) or []
    )
    return columns - continuum_exclude - _AUDIT_FIELDS - audit - special


@dataclass(frozen=True)
class ChangeRecord:
    """One atomic change, as stored in ``version_changes``.

    Fields match the ``version_changes`` columns one-to-one so the
    capture listener can serialise a list of these to
    ``session.bulk_insert_mappings`` without translation.
    """

    kind: str
    path: list[Any]
    from_value: Any
    to_value: Any


Key = str | int


def _values_equivalent(from_value: Any, to_value: Any) -> bool:
    """True if a transition from ``from_value`` to ``to_value`` should
    NOT produce a record.

    Beyond plain ``==`` equality, treats ``None`` and ``""`` as equivalent:
    Superset's save paths normalize nullable strings to ``""`` on first
    write (e.g. ``Dashboard.css``, ``certified_by``,
    ``certification_details``), so a first-save transition between
    null and empty string carries no user-authored signal.
    """
    if from_value == to_value:
        return True
    if from_value in (None, "") and to_value in (None, ""):
        return True
    return False


def _diff_scalar(
    field_name: str,
    from_value: Any,
    to_value: Any,
) -> ChangeRecord | None:
    """Emit a generic ``kind="field"`` record when a scalar differs."""
    if _values_equivalent(from_value, to_value):
        return None
    return ChangeRecord(
        kind="field",
        path=[field_name],
        from_value=from_value,
        to_value=to_value,
    )


def _diff_list_by_natural_key(
    kind: str,
    path_prefix: list[Any],
    from_list: list[Any] | None,
    to_list: list[Any] | None,
    key_fn: Callable[[Any], Key | None],
) -> list[ChangeRecord]:
    """Diff two lists, matching elements by natural key.

    Emits one record per add / remove / modify. When ``key_fn`` returns
    ``None`` for an item (natural key missing or empty), the item falls
    back to its position as a synthetic key — so insertions in the
    middle of a keyless list still produce sensible records, at the
    cost of position-dependent identity.
    """
    from_list = from_list or []
    to_list = to_list or []

    def _effective_key(raw: Key | None, idx: int) -> Key:
        if raw is None or raw == "":
            return idx
        return raw

    from_by_key: dict[Key, Any] = {}
    for idx, item in enumerate(from_list):
        from_by_key[_effective_key(key_fn(item), idx)] = item
    to_by_key: dict[Key, Any] = {}
    for idx, item in enumerate(to_list):
        to_by_key[_effective_key(key_fn(item), idx)] = item

    records: list[ChangeRecord] = []
    # Preserve `from` order then append `to`-only keys, so sequence is
    # deterministic across runs. For dict items (dataset columns /
    # metrics) we strip audit/implementation fields before comparing
    # AND before emitting — otherwise a save that only adds a new
    # column would also emit "changed" records for every existing
    # column, because their ``changed_on`` timestamps all refreshed.
    # The stripped from/to are what the renderer sees; the per-column
    # audit trail is already aggregated at the transaction level in
    # ``version_transaction`` (``user_id`` + ``issued_at``).
    for k, from_item in from_by_key.items():
        to_item = to_by_key.get(k)
        stripped_from = _strip_opaque_fields(from_item)
        if to_item is None:
            records.append(
                ChangeRecord(
                    kind=kind,
                    path=[*path_prefix, k],
                    from_value=stripped_from,
                    to_value=None,
                )
            )
            continue
        stripped_to = _strip_opaque_fields(to_item)
        if stripped_from != stripped_to:
            records.append(
                ChangeRecord(
                    kind=kind,
                    path=[*path_prefix, k],
                    from_value=stripped_from,
                    to_value=stripped_to,
                )
            )
    for k, to_item in to_by_key.items():
        if k not in from_by_key:
            records.append(
                ChangeRecord(
                    kind=kind,
                    path=[*path_prefix, k],
                    from_value=None,
                    to_value=_strip_opaque_fields(to_item),
                )
            )
    return records


def _filter_key(f: Any) -> Key | None:
    """Natural key for an adhoc filter — its subject (column name).

    Users rarely have two filters on the same column; when they do the
    secondary dimensions (operator, comparator) appear in the record's
    from/to values so the renderer can disambiguate.
    """
    return f.get("subject") if isinstance(f, dict) else None


def _metric_key(m: Any) -> Key | None:
    """Natural key for a metric: prefer ``label``, fall back to column+aggregate."""
    if not isinstance(m, dict):
        return None
    if label := m.get("label"):
        return label
    column = m.get("column")
    col_name = column.get("column_name") if isinstance(column, dict) else None
    agg = m.get("aggregate")
    if col_name and agg:
        return f"{agg}({col_name})"
    return None


def _dimension_key(d: Any) -> Key | None:
    """Natural key for a groupby/columns element — usually a bare string."""
    if isinstance(d, str):
        return d
    if isinstance(d, dict):
        return d.get("label") or d.get("column_name")
    return None


def _coerce_params(p: Any) -> dict[str, Any]:
    """Decode ``Slice.params`` which is stored as a JSON string."""
    if p is None:
        return {}
    if isinstance(p, str):
        try:
            decoded = _json.loads(p)
        except _json.JSONDecodeError:
            return {}
        return decoded if isinstance(decoded, dict) else {}
    if isinstance(p, dict):
        return p
    return {}


def diff_slice_params(
    from_params: Any,
    to_params: Any,
) -> list[ChangeRecord]:
    """Diff the ``Slice.params`` JSON blob, promoting known keys to kinds."""
    from_p = _coerce_params(from_params)
    to_p = _coerce_params(to_params)
    records: list[ChangeRecord] = []
    all_keys = (set(from_p) | set(to_p)) - _CHART_PARAMS_AUDIT_KEYS
    for key in sorted(all_keys):
        from_v = from_p.get(key)
        to_v = to_p.get(key)
        if _values_equivalent(from_v, to_v):
            continue
        kind = _CHART_PARAMS_KIND_BY_KEY.get(key)
        if kind == "filter" and isinstance(from_v, list) and isinstance(to_v, list):
            records.extend(
                _diff_list_by_natural_key(
                    "filter",
                    ["params", "adhoc_filters"],
                    from_v,
                    to_v,
                    _filter_key,
                )
            )
        elif kind == "metric" and isinstance(from_v, list) and isinstance(to_v, list):
            records.extend(
                _diff_list_by_natural_key(
                    "metric",
                    ["params", "metrics"],
                    from_v,
                    to_v,
                    _metric_key,
                )
            )
        elif (
            kind == "dimension" and isinstance(from_v, list) and isinstance(to_v, list)
        ):
            records.extend(
                _diff_list_by_natural_key(
                    "dimension",
                    ["params", key],
                    from_v,
                    to_v,
                    _dimension_key,
                )
            )
        elif kind:
            # scalar first-class kind (time_range, color_palette) —
            # single record carrying the whole value
            records.append(
                ChangeRecord(
                    kind=kind,
                    path=["params", key],
                    from_value=from_v,
                    to_value=to_v,
                )
            )
        else:
            # unknown params sub-key: generic field change
            records.append(
                ChangeRecord(
                    kind="field",
                    path=["params", key],
                    from_value=from_v,
                    to_value=to_v,
                )
            )
    return records


def diff_scalar_fields(
    pre: dict[str, Any],
    post: dict[str, Any],
    *,
    fields: Iterable[str],
) -> list[ChangeRecord]:
    """Emit one ``kind="field"`` record per differing field in ``fields``.

    The ``fields`` iterable is supplied by the caller — typically
    :func:`scalar_fields_for` at listener wiring time. Keeping the
    field list outside this function means adding a new column to a
    model does not require a matching edit here.
    """
    records: list[ChangeRecord] = []
    for field in sorted(fields):
        record = _diff_scalar(field, pre.get(field), post.get(field))
        if record is not None:
            records.append(record)
    return records


def diff_slice(
    pre: dict[str, Any],
    post: dict[str, Any],
    *,
    fields: Iterable[str],
) -> list[ChangeRecord]:
    """Full Slice (chart) diff — scalars plus params classification.

    Pass ``fields=scalar_fields_for(Slice, special=frozenset({"params"}))``
    to get the ``params``-excluded scalar set; ``Slice.params`` is diffed
    separately by :func:`diff_slice_params` for kind promotion.
    """
    records = diff_scalar_fields(pre, post, fields=fields)
    records.extend(diff_slice_params(pre.get("params"), post.get("params")))
    return records


def diff_json_field(
    field_name: str,
    from_value: Any,
    to_value: Any,
    *,
    exclude_keys: frozenset[str] = frozenset(),
) -> list[ChangeRecord]:
    """Diff a TEXT column that stores a JSON dict, emitting one record
    per top-level key whose value changed.

    Used for ``Dashboard.json_metadata`` (``position_json`` has its
    own structural diff via :func:`diff_dashboard_layout`). Saving the
    blob verbatim into ``from_value`` / ``to_value`` would swamp the
    change log with multi-KB strings on every save; walking the parsed
    dict at the top level reduces noise to "what changed".

    *exclude_keys* names sub-keys that are frontend-derived /
    auto-stamped on save and don't carry user-authored signal. Same
    rationale as the ``audit`` parameter on
    :func:`scalar_fields_for` for the parent-column level.

    Path is ``[field_name, key]``, mirroring ``diff_slice_params``'s
    ``["params", key]`` shape so renderers can use a single addressing
    scheme across the chart and dashboard sides.
    """
    from_p = _coerce_params(from_value)
    to_p = _coerce_params(to_value)
    records: list[ChangeRecord] = []
    for key in sorted(set(from_p) | set(to_p)):
        if key in exclude_keys:
            continue
        from_v = from_p.get(key)
        to_v = to_p.get(key)
        if _values_equivalent(from_v, to_v):
            continue
        records.append(
            ChangeRecord(
                kind="field",
                path=[field_name, key],
                from_value=from_v,
                to_value=to_v,
            )
        )
    return records


# json_metadata sub-keys that the frontend auto-stamps / auto-derives
# on save. They mirror dashboard membership and chart inventory, not
# user-authored content, so they noise up the change log without
# carrying intent. The records produced for these keys can be ~50KB
# (full label-colour dict) for a one-chart save.
#
#   chart_configuration:        per-chart cross-filter scope state,
#                               re-derived when charts are added/removed.
#   global_chart_configuration: dashboard-wide filter scope; the
#                               ``chartsInScope`` list mirrors live
#                               dashboard membership.
#   map_label_colors:           label → colour map, re-stamped on save
#                               from currently-visible filter values.
#   show_chart_timestamps:      frontend toggle, defaults applied on
#                               save when missing.
#   color_namespace:            scoped colour-scheme namespace, frontend-
#                               derived from the chart set.
DASHBOARD_JSON_METADATA_AUDIT_KEYS: frozenset[str] = frozenset(
    {
        "chart_configuration",
        "global_chart_configuration",
        "map_label_colors",
        "show_chart_timestamps",
        "color_namespace",
    }
)


# Layout component types and how they map to record ``kind`` strings.
# ``HEADER_ID`` is excluded — that's the dashboard's title bar, mirrored
# from ``dashboard_title``. ``ROOT_ID`` and ``GRID_ID`` are structural
# singletons whose only deltas are children lists, which we infer from
# the moves of the children themselves.
_LAYOUT_TYPE_TO_KIND: dict[str, str] = {
    "CHART": "chart",
    "ROW": "row",
    "COLUMN": "column",
    "TAB": "tab",
    "TABS": "tabs",
    "HEADER": "header",
    "MARKDOWN": "markdown",
    "DIVIDER": "divider",
}

# Layout components we never emit records for: ROOT_ID is the layout
# root (always present, never moves); GRID_ID is the singleton vertical
# stack inside ROOT_ID; HEADER_ID is the dashboard's title bar (already
# covered by the ``dashboard_title`` scalar field).
_LAYOUT_SUPPRESSED_IDS: frozenset[str] = frozenset({"ROOT_ID", "GRID_ID", "HEADER_ID"})


def _layout_component_label(node: dict[str, Any]) -> str | None:
    """Extract a human-readable label from a layout node, when one
    exists. Used to build the ``from_value`` / ``to_value`` payload so
    the UI can render messages like "Added chart 'Foo'" without
    needing to fetch related entities.
    """
    meta = node.get("meta") or {}
    if not isinstance(meta, dict):
        return None
    for key in ("sliceName", "label", "text"):
        value = meta.get(key)
        if isinstance(value, str) and value.strip():
            return value
    return None


def _layout_node_payload(node: dict[str, Any]) -> dict[str, Any]:
    """Minimal payload describing a layout node — enough for the UI
    to render the change without dragging the full layout snippet
    (which can be ~1KB per row when CHART nodes carry colour configs).
    """
    meta = node.get("meta") or {}
    if not isinstance(meta, dict):
        meta = {}
    payload: dict[str, Any] = {"id": node.get("id"), "type": node.get("type")}
    if (label := _layout_component_label(node)) is not None:
        payload["name"] = label
    if (chart_id := meta.get("chartId")) is not None:
        payload["chartId"] = chart_id
    # ``uuid`` (slice uuid for CHART nodes) lets the M2M-vs-layout
    # dedupe in :func:`fold_dashboard_layout_with_chart_changes`
    # match on the same key — :func:`diff_dashboard_slices` keys its
    # records by uuid, not chartId.
    if (slice_uuid := meta.get("uuid")) is not None:
        payload["uuid"] = slice_uuid
    return payload


def _layout_parent_id(node: dict[str, Any]) -> Any:
    """The immediate-parent node id for a layout component — the last
    entry in ``parents``. Used to detect moves: same id, different
    parent."""
    parents = node.get("parents") or []
    if not isinstance(parents, list) or not parents:
        return None
    return parents[-1]


def _meta_excluding_position(node: dict[str, Any]) -> dict[str, Any]:
    """Meta dict with ``parents``-equivalent positional bits removed
    so two nodes that differ ONLY in where they sit compare equal at
    the meta level. Move detection uses ``parents`` directly; this is
    for "edit" (meta change) detection."""
    meta = node.get("meta") or {}
    return dict(meta) if isinstance(meta, dict) else {}


def _diff_layout_node(
    node_id: str,
    pre_node: Optional[dict[str, Any]],
    post_node: Optional[dict[str, Any]],
) -> Optional[ChangeRecord]:
    """Diff one component slot in the layout dict and return a record for
    the logical action — add, remove, move, edit — or ``None`` when the
    slot is unchanged or holds an unknown component type.
    """
    node_for_kind = post_node or pre_node or {}
    kind = _LAYOUT_TYPE_TO_KIND.get(node_for_kind.get("type") or "")
    if kind is None:
        return None  # unknown component type — skip rather than emit garbage

    if pre_node is None and post_node is not None:
        return ChangeRecord(
            kind=kind,
            path=["add", kind, node_id],
            from_value=None,
            to_value=_layout_node_payload(post_node),
        )
    if post_node is None and pre_node is not None:
        return ChangeRecord(
            kind=kind,
            path=["remove", kind, node_id],
            from_value=_layout_node_payload(pre_node),
            to_value=None,
        )

    # Both present — check move first, then edit.
    assert pre_node is not None
    assert post_node is not None
    pre_parent = _layout_parent_id(pre_node)
    if pre_parent != (post_parent := _layout_parent_id(post_node)):
        return ChangeRecord(
            kind=kind,
            path=["move", kind, node_id],
            from_value={**_layout_node_payload(pre_node), "parent": pre_parent},
            to_value={**_layout_node_payload(post_node), "parent": post_parent},
        )

    pre_meta = _meta_excluding_position(pre_node)
    if pre_meta != (post_meta := _meta_excluding_position(post_node)):
        return ChangeRecord(
            kind=kind,
            path=["edit", kind, node_id],
            from_value={**_layout_node_payload(pre_node), "meta": pre_meta},
            to_value={**_layout_node_payload(post_node), "meta": post_meta},
        )
    return None


def diff_dashboard_layout(
    pre: Any,
    post: Any,
) -> list[ChangeRecord]:
    """Structural diff of a dashboard's ``position_json``, emitting one
    record per logical layout action.

    Walks both sides keyed on the component ``id`` (e.g.
    ``"CHART-mkPZLOnWCElgL0Udp1gVK"``):

    * id present only in *post* → ``op=add``, ``from_value=None``,
      ``to_value=<minimal payload>``
    * id present only in *pre*  → ``op=remove``, payload swapped
    * id in both, ``parents`` differs → ``op=move``, payloads carry
      old + new parent
    * id in both, parents equal, ``meta`` differs → ``op=edit``,
      payloads carry old + new meta
    * id in both, equal → no record

    The ``operation_type``-style verb is encoded in
    ``path[0]`` as ``["add"|"remove"|"move"|"edit", <component-kind>,
    <component-id>]`` so the UI's path-based renderer can read it
    without inspecting from/to.

    ``ROOT_ID`` / ``GRID_ID`` / ``HEADER_ID`` are suppressed (see
    :data:`_LAYOUT_SUPPRESSED_IDS`).
    """
    pre_nodes = _layout_nodes(pre)
    post_nodes = _layout_nodes(post)
    records: list[ChangeRecord] = []
    for node_id in sorted(set(pre_nodes) | set(post_nodes)):
        record = _diff_layout_node(
            node_id, pre_nodes.get(node_id), post_nodes.get(node_id)
        )
        if record is not None:
            records.append(record)
    return records


def _layout_nodes(raw: Any) -> dict[str, dict[str, Any]]:
    """Coerce *raw* (a ``position_json`` blob or already-parsed dict) into
    the ``{node_id: node_dict}`` shape used by the layout diff, filtering
    out non-dict values and the always-present root/grid/header singletons.
    """
    parsed = _coerce_params(raw)
    return {
        k: v
        for k, v in parsed.items()
        if isinstance(v, dict) and k not in _LAYOUT_SUPPRESSED_IDS
    }


def diff_dashboard(
    pre: dict[str, Any],
    post: dict[str, Any],
    *,
    fields: Iterable[str],
) -> list[ChangeRecord]:
    """Dashboard diff: scalar fields plus structural diff of
    ``json_metadata`` and ``position_json``.

    Promoting ``position_json`` to ``kind="layout"`` or
    ``json_metadata.native_filter_configuration`` to ``kind="filter"``
    is deferred to Phase 2 alongside the UI that would render them
    (spec Clarifications §Session 2026-04-24); until then, both fields
    fall through to ``kind="field"`` records keyed by sub-key.
    """
    records = diff_scalar_fields(pre, post, fields=fields)
    records.extend(
        diff_json_field(
            "json_metadata",
            pre.get("json_metadata"),
            post.get("json_metadata"),
            exclude_keys=DASHBOARD_JSON_METADATA_AUDIT_KEYS,
        )
    )
    records.extend(
        diff_dashboard_layout(pre.get("position_json"), post.get("position_json"))
    )
    return records


def _layout_chart_uuids_by_verb(
    records: list[ChangeRecord],
) -> tuple[set[Any], set[Any]]:
    """Scan *records* for layout ``add``/``remove`` records on charts and
    return ``(added_uuids, removed_uuids)`` sets.
    """
    added: set[Any] = set()
    removed: set[Any] = set()
    for r in records:
        if r.kind != "chart" or len(r.path) < 3:
            continue
        verb = r.path[0]
        if verb == "add" and isinstance(r.to_value, dict):
            uuid_ = r.to_value.get("uuid")
            if uuid_ is not None:
                added.add(uuid_)
        elif verb == "remove" and isinstance(r.from_value, dict):
            uuid_ = r.from_value.get("uuid")
            if uuid_ is not None:
                removed.add(uuid_)
    return added, removed


def _is_redundant_m2m_chart_record(
    r: ChangeRecord, added_uuids: set[Any], removed_uuids: set[Any]
) -> bool:
    """Return ``True`` when *r* is an M2M-style slice record that
    duplicates an already-captured layout add/remove for the same uuid.

    M2M slice records have path ``["slices", uuid]`` (length 2); their
    info is strictly less than the corresponding layout record's
    (no name, no parent), so the layout side wins on dedup.
    """
    if r.kind != "chart" or len(r.path) != 2 or r.path[0] != "slices":
        return False
    slice_uuid = r.path[1]
    if r.from_value is None and r.to_value is not None:
        return slice_uuid in added_uuids
    if r.to_value is None and r.from_value is not None:
        return slice_uuid in removed_uuids
    return False


def fold_dashboard_layout_with_chart_changes(
    records: list[ChangeRecord],
) -> list[ChangeRecord]:
    """When a dashboard save adds/removes charts, the ``slices`` M2M
    diff and the layout diff each emit a record for the same logical
    action. Drop the M2M ``kind="chart"`` records — the layout-side
    record carries more information (chart name, parent container).

    The matching is by slice uuid: ``diff_dashboard_slices`` produces
    records with path ``["slices", <slice-uuid>]``; the layout
    payloads carry the same uuid (sourced from
    ``position_json.CHART-x.meta.uuid``). We dedupe on that key.

    Called from the change-records listener after the M2M and layout
    diffs are both merged into the per-entity buffer.
    """
    added_uuids, removed_uuids = _layout_chart_uuids_by_verb(records)
    return [
        r
        for r in records
        if not _is_redundant_m2m_chart_record(r, added_uuids, removed_uuids)
    ]


def diff_dataset(
    pre: dict[str, Any],
    post: dict[str, Any],
    *,
    fields: Iterable[str],
) -> list[ChangeRecord]:
    """SqlaTable scalar-field diff. All paths emit ``kind="field"``.

    Children (columns, metrics) are diffed separately via
    :func:`diff_dataset_columns` / :func:`diff_dataset_metrics`. The
    listener reads them from Continuum shadow tables
    (``table_columns_version`` / ``sql_metrics_version``) rather than
    walking the ORM collection.
    """
    return diff_scalar_fields(pre, post, fields=fields)


def diff_dataset_columns(
    from_columns: list[dict[str, Any]] | None,
    to_columns: list[dict[str, Any]] | None,
) -> list[ChangeRecord]:
    """Child-collection diff on TableColumn rows, keyed by column_name."""
    return _diff_list_by_natural_key(
        kind="column",
        path_prefix=["columns"],
        from_list=from_columns,
        to_list=to_columns,
        key_fn=lambda c: c.get("column_name") if isinstance(c, dict) else None,
    )


def diff_dataset_metrics(
    from_metrics: list[dict[str, Any]] | None,
    to_metrics: list[dict[str, Any]] | None,
) -> list[ChangeRecord]:
    """Child-collection diff on SqlMetric rows, keyed by metric_name."""
    return _diff_list_by_natural_key(
        kind="metric",
        path_prefix=["metrics"],
        from_list=from_metrics,
        to_list=to_metrics,
        key_fn=lambda m: m.get("metric_name") if isinstance(m, dict) else None,
    )


def diff_dashboard_slices(
    from_slice_uuids: list[str] | None,
    to_slice_uuids: list[str] | None,
) -> list[ChangeRecord]:
    """Diff a dashboard's chart membership, keyed by slice uuid.

    Pure set-diff: added uuids get ``from_value=None, to_value=uuid``;
    removed uuids get the inverse. No "changed" case because chart
    associations are identity-only (the list element IS the uuid).
    """
    from_set = set(from_slice_uuids or [])
    to_set = set(to_slice_uuids or [])
    records: list[ChangeRecord] = []
    for uuid_ in sorted(from_set - to_set):
        records.append(
            ChangeRecord(
                kind="chart",
                path=["slices", uuid_],
                from_value=uuid_,
                to_value=None,
            )
        )
    for uuid_ in sorted(to_set - from_set):
        records.append(
            ChangeRecord(
                kind="chart",
                path=["slices", uuid_],
                from_value=None,
                to_value=uuid_,
            )
        )
    return records