fix(importers): catch JSONDecodeError in load_configs masked_encrypted_extra merge

load_configs() parses each config's `masked_encrypted_extra` field with
json.loads() before schema validation runs, in order to merge caller-supplied
`encrypted_extra_secrets` into it. That field comes straight from user-uploaded
import YAML, so a malformed value raised a raw simplejson.JSONDecodeError (a
ValueError, not a marshmallow.ValidationError). The enclosing except only
catches ValidationError, so the decode error escaped uncaught out of
ImportModelsCommand.validate() and surfaced as an opaque 500 instead of the
structured 422 every other per-file validation failure produces.

Add a sibling `except json.JSONDecodeError` clause that converts the decode
error into a ValidationError with the same {file_name: {field: [msg]}} shape,
so it flows into the existing exceptions list and downstream
CommandInvalidError aggregation. Additive catch only; existing success paths
are unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

docs/admin_docs/configuration/configuring-superset.mdx

@@ -97,6 +97,54 @@ for more information on how to configure it.
 
 At the very least, you'll want to change `SECRET_KEY` and `SQLALCHEMY_DATABASE_URI`. Continue reading for more about each of these.
 
+## Localizing D3 date and time labels
+
+`BABEL_DEFAULT_LOCALE` controls Superset's application translations, while
+`D3_TIME_FORMAT` provides localized date and time names to visualizations that
+use the D3 formatter registry, including Calendar Heatmap. Configure both when
+you want the application and chart labels to use the same locale.
+
+`D3_TIME_FORMAT` accepts partial overrides. For example, Russian month names
+can be configured in `superset_config.py` as follows:
+
+```python
+BABEL_DEFAULT_LOCALE = "ru"
+
+D3_TIME_FORMAT = {
+    "months": [
+        "Январь",
+        "Февраль",
+        "Март",
+        "Апрель",
+        "Май",
+        "Июнь",
+        "Июль",
+        "Август",
+        "Сентябрь",
+        "Октябрь",
+        "Ноябрь",
+        "Декабрь",
+    ],
+    "shortMonths": [
+        "Янв",
+        "Фев",
+        "Мар",
+        "Апр",
+        "Май",
+        "Июн",
+        "Июл",
+        "Авг",
+        "Сен",
+        "Окт",
+        "Ноя",
+        "Дек",
+    ],
+}
+```
+
+Restart Superset after changing `superset_config.py` so the frontend receives
+the updated formatter configuration.
+
 ## Chart-data query timing
 
 Set `CHART_DATA_INCLUDE_TIMING = True` to add an optional versioned timing object

superset-frontend/plugins/plugin-chart-calendar/src/Calendar.ts

@@ -22,7 +22,7 @@ import { getSequentialSchemeRegistry } from '@superset-ui/core';
 import { SupersetTheme } from '@apache-superset/core/theme';
 import { t } from '@apache-superset/core/translation';
 import CalHeatMapImport from './vendor/cal-heatmap';
-import { convertUTCTimestampToLocal } from './utils';
+import { convertUTCTimestampToLocal, getFormattedUTCTime } from './utils';
 
 // The vendor file is @ts-nocheck, so its export lacks type info.
 // Define a minimal constructor interface for use in this file.
@@ -103,6 +103,8 @@ function Calendar(element: HTMLElement, props: CalendarProps) {
   const subDomainTextFormat = showValues
     ? (_date: Date, value: number) => valueFormatter(value)
     : null;
+  const dateFormatter = (date: Date, format: string) =>
+    getFormattedUTCTime(date.getTime(), format);
 
   const metricsData = data.data;
 
@@ -166,6 +168,7 @@ function Calendar(element: HTMLElement, props: CalendarProps) {
       itemName: '',
       valueFormatter,
       timeFormatter,
+      dateFormatter,
       subDomainTextFormat,
     });
   });

superset-frontend/plugins/plugin-chart-calendar/src/vendor/cal-heatmap.ts

@@ -76,6 +76,8 @@ var CalHeatMap = function () {
 
     timeFormatter: d => d,
 
+    dateFormatter: null,
+
     domain: 'hour',
 
     subDomain: 'min',
@@ -1990,10 +1992,14 @@ CalHeatMap.prototype = {
 
     if (typeof format === 'function') {
       return format(d);
-    } else {
-      var f = d3.time.format(format);
-      return f(d);
     }
+
+    if (typeof this.options.dateFormatter === 'function') {
+      return this.options.dateFormatter(d, format);
+    }
+
+    var f = d3.time.format(format);
+    return f(d);
   },
 
   getSubDomainTitle: function (d) {

superset-frontend/plugins/plugin-chart-calendar/test/ReactCalendar.test.tsx

@@ -25,9 +25,11 @@ import {
   waitFor,
 } from 'spec/helpers/testing-library';
 import { CALENDAR_TOOLTIP_CLASS } from '../src/tooltip';
+import { convertUTCTimestampToLocal } from '../src/utils';
 
 interface MockCalHeatMapConfig {
   itemSelector: Element;
+  dateFormatter?: (date: Date, format: string) => string;
 }
 
 type MetricNameInput = string | string[];
@@ -38,6 +40,7 @@ let mockInitCallCount = 0;
 let mockThrowOnInitCall: number | null = null;
 let mockDestroyCallCount = 0;
 let mockDestroyedInstanceIds: string[] = [];
+let mockDateFormatter: MockCalHeatMapConfig['dateFormatter'];
 
 const mockTheme = {
   colorBgElevated: '#ffffff',
@@ -56,6 +59,7 @@ jest.mock('../src/vendor/cal-heatmap', () => ({
       } = require('../src/tooltip');
 
       mockInitCallCount += 1;
+      mockDateFormatter = config.dateFormatter;
       if (mockThrowOnInitCall === mockInitCallCount) {
         throw new Error('Mock CalHeatMap init failure');
       }
@@ -284,9 +288,28 @@ afterEach(() => {
   mockThrowOnInitCall = null;
   mockDestroyCallCount = 0;
   mockDestroyedInstanceIds = [];
+  mockDateFormatter = undefined;
   document.body.innerHTML = '';
 });
 
+test('Calendar provides a timezone-safe date formatter to CalHeatMap', () => {
+  const calendarOwner = document.createElement('div');
+  document.body.appendChild(calendarOwner);
+
+  Calendar(calendarOwner, {
+    ...createCalendarProps('localized-metric'),
+    theme: mockTheme,
+  });
+
+  if (!mockDateFormatter) {
+    throw new Error('Expected Calendar to configure a date formatter');
+  }
+
+  const localDate = new Date(convertUTCTimestampToLocal(Date.UTC(2024, 0, 1)));
+
+  expect(mockDateFormatter(localDate, '%Y-%m-%d')).toBe('2024-01-01');
+});
+
 test('rerender and unmount clean up only the affected calendar tooltips', () => {
   jest.useFakeTimers();
 

superset-frontend/plugins/plugin-chart-calendar/test/cal-heatmap.test.ts

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import CalHeatMapImport from '../src/vendor/cal-heatmap';
+
+type DateFormatter = (date: Date, format: string) => string;
+type FunctionalDateFormat = (date: Date) => string;
+
+interface CalHeatMapInstance {
+  options: {
+    dateFormatter: DateFormatter | null;
+  };
+  formatDate(date: Date, format: string | FunctionalDateFormat): string;
+}
+
+const CalHeatMap = CalHeatMapImport as unknown as new () => CalHeatMapInstance;
+
+test('CalHeatMap delegates string date formats to the configured formatter', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+  const dateFormatter = jest.fn<string, [Date, string]>(() => 'Январь');
+  calendar.options.dateFormatter = dateFormatter;
+
+  expect(calendar.formatDate(date, '%B')).toBe('Январь');
+  expect(dateFormatter).toHaveBeenCalledWith(date, '%B');
+});
+
+test('CalHeatMap preserves functional formatters over the configured formatter', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+  const dateFormatter = jest.fn<string, [Date, string]>(() => 'localized');
+  const functionalFormat = jest.fn<string, [Date]>(() => 'custom');
+  calendar.options.dateFormatter = dateFormatter;
+
+  expect(calendar.formatDate(date, functionalFormat)).toBe('custom');
+  expect(functionalFormat).toHaveBeenCalledWith(date);
+  expect(dateFormatter).not.toHaveBeenCalled();
+});
+
+test('CalHeatMap keeps the D3 formatter fallback', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+
+  expect(calendar.formatDate(date, '%B')).toBe('January');
+});

superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx

@@ -867,7 +867,7 @@ function DatasourceEditor({
       return {
         ...metric,
         certification_details: certificationDetails || details,
-        warning_markdown: warningMarkdown || '',
+        warning_markdown: warningMarkdown || metric.warning_markdown || '',
         certified_by: certifiedBy || certifiedByMetric,
       };
     }),

superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditorMetricCertification.test.tsx

@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fetchMock from 'fetch-mock';
+import { screen, userEvent, waitFor } from 'spec/helpers/testing-library';
+import {
+  createProps,
+  DATASOURCE_ENDPOINT,
+  setupDatasourceEditorMocks,
+  cleanupAsyncOperations,
+  fastRender,
+  dismissDatasourceWarning,
+} from './DatasourceEditor.test.utils';
+
+beforeEach(() => {
+  fetchMock.get(DATASOURCE_ENDPOINT, [], { name: DATASOURCE_ENDPOINT });
+  setupDatasourceEditorMocks();
+});
+
+afterEach(async () => {
+  await cleanupAsyncOperations();
+  fetchMock.clearHistory().removeRoutes();
+});
+
+// Certifying a metric fills two adjacent fields in one visit to the expanded
+// row. Both are committed through TextControl's debounce, so the second one
+// used to land on the item as it looked before the first had been applied,
+// leaving the saved metric with details but no certifier.
+test('certifying a metric keeps both certified_by and certification_details', async () => {
+  const testProps = createProps();
+  fastRender(testProps);
+  await dismissDatasourceWarning();
+
+  await userEvent.click(await screen.findByTestId('collection-tab-Metrics'));
+  const expandToggles = await screen.findAllByLabelText(/expand row/i);
+  await userEvent.click(expandToggles[0]);
+
+  await userEvent.type(
+    await screen.findByPlaceholderText('Certified by'),
+    'Metric Certifier',
+  );
+  await userEvent.type(
+    await screen.findByPlaceholderText('Certification details'),
+    'Metric cert details',
+  );
+
+  await waitFor(() => {
+    const { calls } = testProps.onChange.mock;
+    const savedMetrics = calls[calls.length - 1]?.[0]?.metrics ?? [];
+    const saved = savedMetrics.find(metric => metric.metric_name === 'count');
+    expect(saved).toEqual(
+      expect.objectContaining({
+        certified_by: 'Metric Certifier',
+        certification_details: 'Metric cert details',
+      }),
+    );
+  });
+});

superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditorWarningMarkdown.test.tsx

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fetchMock from 'fetch-mock';
+import { screen, userEvent } from 'spec/helpers/testing-library';
+import {
+  createProps,
+  DATASOURCE_ENDPOINT,
+  setupDatasourceEditorMocks,
+  cleanupAsyncOperations,
+  fastRender,
+  dismissDatasourceWarning,
+} from './DatasourceEditor.test.utils';
+
+// Stub the Ace-backed control with a plain textarea. Ace spreads its document
+// across many spans and keeps only the keystroke buffer in its own textarea,
+// so asserting on the value the control receives is less brittle than
+// reaching into Ace's DOM.
+jest.mock('src/explore/components/controls/TextAreaControl', () => ({
+  __esModule: true,
+  default: ({
+    controlId,
+    value,
+    onChange,
+  }: {
+    controlId?: string;
+    value?: string;
+    onChange?: (value: string) => void;
+  }) => (
+    <textarea
+      data-test={`mock-textarea-${controlId}`}
+      value={value ?? ''}
+      onChange={event => onChange?.(event.target.value)}
+    />
+  ),
+}));
+
+beforeEach(() => {
+  fetchMock.get(DATASOURCE_ENDPOINT, [], { name: DATASOURCE_ENDPOINT });
+  setupDatasourceEditorMocks();
+});
+
+afterEach(async () => {
+  await cleanupAsyncOperations();
+  fetchMock.clearHistory().removeRoutes();
+});
+
+// Regression test for #42704. Explore's datasource payload (SqlMetric.data on
+// the backend) exposes warning_markdown as a flattened top-level field and
+// omits the raw `extra` JSON string that the /api/v1/dataset/{id} endpoint
+// backing the Datasets page provides. Deriving warning_markdown purely from
+// `extra` therefore dropped the saved text when the modal was opened from
+// Explore, leaving the Warning field blank on reopen.
+test('keeps a pre-existing top-level warning_markdown when the metric has no extra', async () => {
+  const baseProps = createProps();
+  const testProps = {
+    ...baseProps,
+    datasource: {
+      ...baseProps.datasource,
+      metrics: [
+        {
+          ...baseProps.datasource.metrics[0],
+          warning_markdown: 'existing warning',
+          extra: undefined,
+        },
+      ],
+    },
+  };
+
+  fastRender(testProps);
+  await dismissDatasourceWarning();
+
+  const metricsTab = await screen.findByTestId('collection-tab-Metrics');
+  await userEvent.click(metricsTab);
+
+  const expandToggles = await screen.findAllByLabelText(/expand row/i);
+  await userEvent.click(expandToggles[0]);
+
+  expect(
+    await screen.findByTestId('mock-textarea-warning_markdown'),
+  ).toHaveValue('existing warning');
+});

superset-frontend/src/components/Datasource/components/Fieldset/index.tsx

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { ReactNode, useCallback } from 'react';
+import { ReactNode, useCallback, useEffect, useRef } from 'react';
 import { Divider, Form, Typography } from '@superset-ui/core/components';
 import { css } from '@apache-superset/core/theme';
 import { recurseReactClone } from '../../utils';
@@ -39,14 +39,24 @@ export default function Fieldset({
   title = null,
   compact = false,
 }: FieldsetProps) {
+  // Controls report their edits asynchronously - TextControl debounces by
+  // FAST_DEBOUNCE - so the callback that eventually fires was built during an
+  // earlier render. Spreading that render's `item` rebuilds the whole record
+  // from a snapshot taken before a sibling field committed, dropping the value
+  // the user typed first. Reading off a ref merges into the latest commit.
+  const itemRef = useRef(item);
+  useEffect(() => {
+    itemRef.current = item;
+  }, [item]);
+
   const handleChange = useCallback(
     (fieldKey: fieldKeyType, val: any) => {
       onChange?.({
-        ...item,
+        ...itemRef.current,
         [fieldKey]: val,
       });
     },
-    [onChange, item],
+    [onChange],
   );
 
   const propExtender = (field: { props: { fieldKey: fieldKeyType } }) => ({

superset-frontend/src/dashboard/actions/nativeFilters.ts

@@ -27,7 +27,7 @@ import {
 } from '@superset-ui/core';
 import { Dispatch } from 'redux';
 import { RootState } from 'src/dashboard/types';
-import { cloneDeep } from 'lodash-es';
+import { cloneDeep, omit } from 'lodash-es';
 import { setDataMaskForFilterChangesComplete } from 'src/dataMask/actions';
 import { HYDRATE_DASHBOARD } from './hydrate';
 import {
@@ -90,12 +90,20 @@ export const setFilterConfiguration =
     });
     try {
       const response = await updateFilters(filterChanges);
+      // chartsInScope/tabsInScope are derived from the live layout, and the
+      // response carries the persisted copy for every filter - including the
+      // ones this save never touched, whose copy is whatever was stored when
+      // the dashboard was last saved. Dropping them lets the reducers keep the
+      // scopes calculateScopes already computed for this session.
+      const savedFilters = response.result.map(
+        filter => omit(filter, ['chartsInScope', 'tabsInScope']) as Filter,
+      );
       dispatch({
         type: SET_NATIVE_FILTERS_CONFIG_COMPLETE,
-        filterChanges: response.result,
+        filterChanges: savedFilters,
         deletedIds: filterChanges.deleted,
       });
-      dispatch(nativeFiltersConfigChanged(response.result));
+      dispatch(nativeFiltersConfigChanged(savedFilters));
       dispatch(setDataMaskForFilterChangesComplete(filterChanges, oldFilters));
     } catch (err) {
       dispatch({

superset/commands/importers/v1/utils.py

@@ -247,6 +247,20 @@ def load_configs(
                     )
                 exc.messages = {file_name: exc.messages}
                 exceptions.append(exc)
+            except json.JSONDecodeError as exc:
+                # masked_encrypted_extra comes straight from the imported YAML
+                # (before schema validation) and may not be valid JSON. Convert
+                # the raw decode error into a ValidationError so it flows into
+                # the aggregated CommandInvalidError like every other per-file
+                # validation failure, instead of escaping as an opaque 500.
+                logger.error(
+                    "Invalid JSON in masked_encrypted_extra for %s: %s",
+                    file_name,
+                    exc,
+                )
+                exceptions.append(
+                    ValidationError({file_name: {"masked_encrypted_extra": [str(exc)]}})
+                )
 
     return configs
 

superset/daos/dashboard.py

@@ -34,6 +34,7 @@ from superset.commands.dashboard.exceptions import (
     DashboardUpdateFailedError,
 )
 from superset.daos.base import BaseDAO, ColumnOperator, ColumnOperatorEnum
+from superset.dashboards.filter_scope import derive_metadata_scopes
 from superset.dashboards.filters import DashboardAccessFilter
 from superset.exceptions import SupersetSecurityException
 from superset.extensions import db
@@ -547,7 +548,9 @@ class DashboardDAO(BaseDAO[Dashboard]):
         cls, id: str
     ) -> dict[str, list[dict[str, Any]]]:
         dashboard = cls.get_by_id_or_slug(id)
-        metadata = json.loads(dashboard.json_metadata or "{}")
+        metadata = derive_metadata_scopes(
+            dashboard, json.loads(dashboard.json_metadata or "{}")
+        )
         native_filter_configuration = metadata.get("native_filter_configuration", [])
 
         tab_filters = defaultdict(list)
@@ -617,6 +620,13 @@ class DashboardDAO(BaseDAO[Dashboard]):
             metadata["native_filter_configuration"] = updated_configuration
             dashboard.json_metadata = json.dumps(metadata)
 
+            # The client rebuilds its in-scope state from this response, so hand
+            # back derived scopes rather than the stored caches, which are stale
+            # for every filter the caller did not touch.
+            updated_configuration = derive_metadata_scopes(dashboard, metadata)[
+                "native_filter_configuration"
+            ]
+
         return updated_configuration
 
     @classmethod

superset/dashboards/api.py

@@ -91,6 +91,7 @@ from superset.commands.importers.v1.utils import get_contents_from_bundle
 from superset.commands.purge import PurgeArchivedCommand, SoftDeleteBinding
 from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod
 from superset.daos.dashboard import DashboardDAO, EmbeddedDashboardDAO
+from superset.dashboards.filter_scope import derive_json_metadata
 from superset.dashboards.filters import (
     DashboardAccessFilter,
     DashboardCertifiedFilter,
@@ -653,6 +654,12 @@ class DashboardRestApi(
             schema = self.dashboard_get_response_schema
 
         result = schema.dump(dash)
+        if json_metadata := result.get("json_metadata"):
+            # The stored scope caches (``chartsInScope``, ``tabsInScope``,
+            # ``chart_configuration``) go stale as soon as the layout changes;
+            # derive them so callers see the same document the dashboard client
+            # computes for itself.
+            result["json_metadata"] = derive_json_metadata(dash, json_metadata)
         if "charts" in result:
             # Only name the member charts the caller can access, consistent with
             # the per-object narrowing applied to the dashboard's datasets and

superset/dashboards/filter_scope.py

@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Derive filter-scope caches in ``json_metadata`` from the dashboard layout.
+
+``chartsInScope`` / ``tabsInScope`` on a native filter, and the ``chartsInScope``
+lists inside ``chart_configuration`` / ``global_chart_configuration``, are
+denormalized caches of the authoritative ``scope`` plus ``position_json``. They
+are written when a dashboard is saved and are never revisited afterwards, so a
+dashboard that has charts added or removed - or that was seeded, exported or
+imported - carries scope arrays naming charts it does not contain.
+
+The dashboard client already ignores the stored values and recomputes them from
+the live layout on every load, which is why the JSON Metadata panel and
+``GET /api/v1/dashboard/{id}`` disagreed on a dashboard nobody had ever saved.
+Deriving them on read makes the API agree with the client and keeps integrations
+that read ``native_filter_configuration`` from receiving dangling chart ids.
+
+The rules mirror the client (``superset-frontend/src/dashboard/util``):
+``calculateScopes``, ``getChartIdsInFilterScope``, ``findTabsWithChartsInScope``
+and ``getCrossFiltersConfiguration``.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, TYPE_CHECKING
+
+from superset.utils import json
+
+if TYPE_CHECKING:
+    from superset.models.dashboard import Dashboard
+
+CHART_TYPE = "CHART"
+TAB_TYPE = "TAB"
+NATIVE_FILTER_DIVIDER_PREFIX = "NATIVE_FILTER_DIVIDER-"
+DIVIDER_TYPES = frozenset({"DIVIDER", "CHART_CUSTOMIZATION_DIVIDER"})
+
+# ``chart-<chartId>-layer-<layerIndex>``, the per-layer scope keys a deck.gl
+# multi-layer chart contributes to ``scope.selectedLayers``.
+LAYER_SELECTION_RE = re.compile(r"^chart-(\d+)-layer-(\d+)$")
+
+ChartLayoutItems = dict[int, list[dict[str, Any]]]
+
+
+def build_chart_layout_items(position_data: dict[str, Any]) -> ChartLayoutItems:
+    """Map each chart id in the layout to the layout items that render it."""
+    chart_layout_items: ChartLayoutItems = {}
+    for item in position_data.values():
+        if not isinstance(item, dict) or item.get("type") != CHART_TYPE:
+            continue
+        chart_id = item.get("meta", {}).get("chartId")
+        if isinstance(chart_id, int):
+            chart_layout_items.setdefault(chart_id, []).append(item)
+    return chart_layout_items
+
+
+def get_chart_ids_in_scope(
+    scope: dict[str, Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[int]:
+    """Charts covered by ``scope``, in ``chart_ids`` order."""
+    excluded = set(scope.get("excluded") or [])
+    root_path = set(scope.get("rootPath") or [])
+
+    def in_scope(chart_id: int) -> bool:
+        if chart_id in excluded:
+            return False
+        return any(
+            parent in root_path
+            for layout_item in chart_layout_items.get(chart_id, [])
+            for parent in layout_item.get("parents") or []
+        )
+
+    selected_layers = scope.get("selectedLayers") or []
+    if not selected_layers:
+        return [chart_id for chart_id in chart_ids if in_scope(chart_id)]
+
+    # A layer selection targets its chart directly, and suppresses the
+    # rootPath/excluded test for that chart.
+    charts_with_layer_selections = set()
+    targeted: list[int] = []
+    chart_id_set = set(chart_ids)
+    for selection_key in selected_layers:
+        if match := LAYER_SELECTION_RE.match(str(selection_key)):
+            chart_id = int(match.group(1))
+            charts_with_layer_selections.add(chart_id)
+            if chart_id in chart_id_set and chart_id not in targeted:
+                targeted.append(chart_id)
+
+    return targeted + [
+        chart_id
+        for chart_id in chart_ids
+        if chart_id not in charts_with_layer_selections
+        and chart_id not in targeted
+        and in_scope(chart_id)
+    ]
+
+
+def get_tabs_in_scope(
+    charts_in_scope: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[str]:
+    """Tabs holding at least one of ``charts_in_scope``."""
+    tabs_in_scope: list[str] = []
+    seen: set[str] = set()
+    for chart_id in charts_in_scope:
+        for layout_item in chart_layout_items.get(chart_id, []):
+            for parent in layout_item.get("parents") or []:
+                if parent.startswith(f"{TAB_TYPE}-") and parent not in seen:
+                    seen.add(parent)
+                    tabs_in_scope.append(parent)
+    return tabs_in_scope
+
+
+def _is_divider(item: dict[str, Any]) -> bool:
+    return (
+        str(item.get("id", "")).startswith(NATIVE_FILTER_DIVIDER_PREFIX)
+        or item.get("type") in DIVIDER_TYPES
+    )
+
+
+def _derive_item_scopes(
+    items: list[Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[Any]:
+    """Restamp ``chartsInScope`` / ``tabsInScope`` on scoped config items.
+
+    Items without a usable ``scope`` are returned untouched: legacy chart
+    customizations target a chart directly and only gain a ``scope`` once the
+    client migrates them, so overwriting their cache here would drop targeting
+    the client still needs.
+    """
+    derived = []
+    for item in items:
+        if not isinstance(item, dict):
+            derived.append(item)
+            continue
+        if _is_divider(item):
+            derived.append({**item, "chartsInScope": [], "tabsInScope": []})
+            continue
+        scope = item.get("scope")
+        if not isinstance(scope, dict) or not isinstance(scope.get("excluded"), list):
+            derived.append(item)
+            continue
+        charts_in_scope = get_chart_ids_in_scope(scope, chart_ids, chart_layout_items)
+        derived.append(
+            {
+                **item,
+                "chartsInScope": charts_in_scope,
+                "tabsInScope": get_tabs_in_scope(charts_in_scope, chart_layout_items),
+            }
+        )
+    return derived
+
+
+def _derive_cross_filter_scopes(
+    metadata: dict[str, Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> None:
+    global_config = metadata.get("global_chart_configuration")
+    global_charts_in_scope = chart_ids
+    if isinstance(global_config, dict) and isinstance(global_config.get("scope"), dict):
+        global_charts_in_scope = get_chart_ids_in_scope(
+            global_config["scope"], chart_ids, chart_layout_items
+        )
+        metadata["global_chart_configuration"] = {
+            **global_config,
+            "chartsInScope": global_charts_in_scope,
+        }
+
+    chart_configuration = metadata.get("chart_configuration")
+    if not isinstance(chart_configuration, dict):
+        return
+
+    derived_configuration = {}
+    for key, config in chart_configuration.items():
+        try:
+            chart_id = int(key)
+        except (TypeError, ValueError):
+            derived_configuration[key] = config
+            continue
+        # Config for a chart no longer on the dashboard is dead weight; the
+        # client drops it on load for the same reason.
+        if chart_id not in chart_layout_items:
+            continue
+        if not isinstance(config, dict):
+            derived_configuration[key] = config
+            continue
+        cross_filters = config.get("crossFilters")
+        if not isinstance(cross_filters, dict):
+            derived_configuration[key] = config
+            continue
+        scope = cross_filters.get("scope")
+        if isinstance(scope, dict):
+            charts_in_scope = get_chart_ids_in_scope(
+                scope, chart_ids, chart_layout_items
+            )
+        else:
+            # Anything that is not an explicit scope object points at the
+            # dashboard-wide scope, which never includes the emitting chart.
+            charts_in_scope = [cid for cid in global_charts_in_scope if cid != chart_id]
+        derived_configuration[key] = {
+            **config,
+            "crossFilters": {**cross_filters, "chartsInScope": charts_in_scope},
+        }
+
+    metadata["chart_configuration"] = derived_configuration
+
+
+def derive_scopes(
+    metadata: dict[str, Any],
+    position_data: dict[str, Any],
+    chart_ids: list[int],
+) -> dict[str, Any]:
+    """Return ``metadata`` with every derived scope cache recomputed.
+
+    ``chart_ids`` orders the resulting ``chartsInScope`` arrays and should be the
+    dashboard's chart ids as the client sees them, so that the API and the JSON
+    Metadata panel produce byte-identical documents.
+    """
+    derived = dict(metadata)
+    chart_layout_items = build_chart_layout_items(position_data)
+
+    for key in ("native_filter_configuration", "chart_customization_config"):
+        config = derived.get(key)
+        if isinstance(config, list):
+            derived[key] = _derive_item_scopes(config, chart_ids, chart_layout_items)
+
+    _derive_cross_filter_scopes(derived, chart_ids, chart_layout_items)
+    return derived
+
+
+def derive_metadata_scopes(
+    dashboard: Dashboard,
+    metadata: dict[str, Any],
+) -> dict[str, Any]:
+    """``derive_scopes`` for a dashboard model's parsed ``json_metadata``."""
+    return derive_scopes(
+        metadata,
+        dashboard.position,
+        [slc.id for slc in dashboard.slices],
+    )
+
+
+def derive_json_metadata(dashboard: Dashboard, json_metadata: str) -> str:
+    """``derive_metadata_scopes`` over a raw ``json_metadata`` string.
+
+    Metadata that does not parse as a JSON object is handed back untouched -
+    reading a dashboard is not the place to start rejecting documents that have
+    always been served as-is.
+    """
+    try:
+        metadata = json.loads(json_metadata)
+    except (TypeError, ValueError):
+        return json_metadata
+    if not isinstance(metadata, dict):
+        return json_metadata
+    return json.dumps(derive_metadata_scopes(dashboard, metadata))

tests/integration_tests/dashboards/api_tests.py

@@ -646,6 +646,65 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
         db.session.delete(dashboard)
         db.session.commit()
 
+    def test_get_dashboard_derives_stale_filter_scope(self):
+        """
+        Dashboard API: ``chartsInScope`` is derived from the layout, not read
+        back from the stored cache (sc-116923).
+        """
+        admin = self.get_user("admin")
+        slices = db.session.query(Slice).limit(2).all()
+        positions = {
+            "ROOT_ID": {"id": "ROOT_ID", "type": "ROOT", "children": ["GRID_ID"]},
+            "GRID_ID": {"id": "GRID_ID", "type": "GRID", "parents": ["ROOT_ID"]},
+        }
+        for slc in slices:
+            positions[f"CHART-{slc.id}"] = {
+                "id": f"CHART-{slc.id}",
+                "type": "CHART",
+                "meta": {"chartId": slc.id},
+                "parents": ["ROOT_ID", "GRID_ID"],
+            }
+        # A scope naming charts the dashboard does not contain - the state every
+        # seeded and imported dashboard starts in.
+        stored_metadata = {
+            "native_filter_configuration": [
+                {
+                    "id": "NATIVE_FILTER-1",
+                    "name": "Region",
+                    "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                    "chartsInScope": [90001, 90002],
+                    "tabsInScope": ["TAB-gone"],
+                }
+            ]
+        }
+        dashboard = self.insert_dashboard(
+            "scope-cache",
+            "scope-cache",
+            [admin.id],
+            slices=slices,
+            position_json=json.dumps(positions),
+            json_metadata=json.dumps(stored_metadata),
+        )
+
+        self.login(ADMIN_USERNAME)
+        rv = self.get_assert_metric(f"api/v1/dashboard/{dashboard.id}", "get")
+        assert rv.status_code == 200
+
+        response_metadata = json.loads(
+            json.loads(rv.data.decode("utf-8"))["result"]["json_metadata"]
+        )
+        native_filter = response_metadata["native_filter_configuration"][0]
+        assert sorted(native_filter["chartsInScope"]) == sorted(
+            slc.id for slc in slices
+        )
+        assert native_filter["tabsInScope"] == []
+        assert native_filter["name"] == "Region"
+        # Deriving is read-only; the stored document is left alone.
+        assert json.loads(dashboard.json_metadata) == stored_metadata
+
+        db.session.delete(dashboard)
+        db.session.commit()
+
     def test_get_dashboard_with_columns(self):
         """
         Dashboard API: Test get dashboard with column selection via q param

tests/unit_tests/commands/importers/v1/utils_test.py

@@ -138,3 +138,105 @@ class TestLoadYaml:
 
         with pytest.raises(ValidationError):
             load_yaml("test.yaml", 'key: "unterminated string')
+
+
+class TestLoadConfigs:
+    """
+    load_configs() merges caller-supplied ``encrypted_extra_secrets`` into the
+    ``masked_encrypted_extra`` field of each config, which comes straight from
+    the imported YAML (before schema validation). A malformed value there used
+    to raise a raw simplejson.JSONDecodeError that escaped uncaught (opaque
+    500); it must instead be collected as a ValidationError like every other
+    per-file failure.
+    """
+
+    @staticmethod
+    def _trivial_schema():  # type: ignore[no-untyped-def]
+        from marshmallow import EXCLUDE, Schema
+
+        class TrivialSchema(Schema):
+            class Meta:
+                unknown = EXCLUDE
+
+        return TrivialSchema()
+
+    @patch("superset.commands.importers.v1.utils.db")
+    def test_invalid_json_in_masked_encrypted_extra_is_collected(
+        self, mock_db: object
+    ) -> None:
+        """A non-JSON ``masked_encrypted_extra`` is converted into a
+        ValidationError appended to ``exceptions`` rather than raising."""
+        from marshmallow.exceptions import ValidationError
+
+        from superset.commands.importers.v1.utils import load_configs
+
+        # No existing databases / ssh tunnels in the (mocked) metadata DB.
+        mock_db.session.query.return_value.all.return_value = []  # type: ignore[attr-defined]
+
+        file_name = "databases/db.yaml"
+        contents = {
+            file_name: (
+                "uuid: abc-123\n"
+                "password: secret\n"
+                "masked_encrypted_extra: not valid json\n"
+            )
+        }
+        exceptions: list[ValidationError] = []
+
+        configs = load_configs(
+            contents=contents,
+            schemas={"databases/": self._trivial_schema()},
+            passwords={},
+            exceptions=exceptions,
+            ssh_tunnel_passwords={},
+            ssh_tunnel_private_keys={},
+            ssh_tunnel_priv_key_passwords={},
+            encrypted_extra_secrets={file_name: {"$.foo": "actual_secret"}},
+        )
+
+        # The bad file is not added to configs, and a structured error is
+        # collected instead of a raw JSONDecodeError propagating out.
+        assert file_name not in configs
+        assert len(exceptions) == 1
+        assert isinstance(exceptions[0], ValidationError)
+        assert file_name in exceptions[0].messages
+        assert "masked_encrypted_extra" in exceptions[0].messages[file_name]
+
+    @patch("superset.commands.importers.v1.utils.db")
+    def test_valid_json_in_masked_encrypted_extra_still_merges(
+        self, mock_db: object
+    ) -> None:
+        """Control: valid JSON in ``masked_encrypted_extra`` still has the
+        secrets merged in and produces no exceptions."""
+        from marshmallow.exceptions import ValidationError
+
+        from superset.commands.importers.v1.utils import load_configs
+        from superset.utils import json
+
+        mock_db.session.query.return_value.all.return_value = []  # type: ignore[attr-defined]
+
+        file_name = "databases/db.yaml"
+        contents = {
+            file_name: (
+                "uuid: abc-123\n"
+                "password: secret\n"
+                'masked_encrypted_extra: \'{"foo": "XXXXXXXXXX"}\'\n'
+            )
+        }
+        exceptions: list[ValidationError] = []
+
+        configs = load_configs(
+            contents=contents,
+            schemas={"databases/": self._trivial_schema()},
+            passwords={},
+            exceptions=exceptions,
+            ssh_tunnel_passwords={},
+            ssh_tunnel_private_keys={},
+            ssh_tunnel_priv_key_passwords={},
+            encrypted_extra_secrets={file_name: {"$.foo": "actual_secret"}},
+        )
+
+        assert exceptions == []
+        assert file_name in configs
+        merged = json.loads(configs[file_name]["masked_encrypted_extra"])
+        assert merged == {"foo": "actual_secret"}

tests/unit_tests/dashboards/filter_scope_test.py

@@ -0,0 +1,233 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from types import SimpleNamespace
+from typing import Any
+
+from superset.dashboards.filter_scope import (
+    derive_json_metadata,
+    derive_metadata_scopes,
+    derive_scopes,
+)
+from superset.utils import json
+
+# Two charts in a tab, one chart outside it.
+POSITION_DATA: dict[str, Any] = {
+    "ROOT_ID": {"id": "ROOT_ID", "type": "ROOT", "children": ["GRID_ID"]},
+    "CHART-outside": {
+        "id": "CHART-outside",
+        "type": "CHART",
+        "meta": {"chartId": 1},
+        "parents": ["ROOT_ID", "GRID_ID"],
+    },
+    "CHART-in-tab": {
+        "id": "CHART-in-tab",
+        "type": "CHART",
+        "meta": {"chartId": 2},
+        "parents": ["ROOT_ID", "GRID_ID", "TABS-1", "TAB-1"],
+    },
+    "CHART-also-in-tab": {
+        "id": "CHART-also-in-tab",
+        "type": "CHART",
+        "meta": {"chartId": 3},
+        "parents": ["ROOT_ID", "GRID_ID", "TABS-1", "TAB-1"],
+    },
+    "MARKDOWN-1": {"id": "MARKDOWN-1", "type": "MARKDOWN", "parents": ["ROOT_ID"]},
+}
+CHART_IDS = [1, 2, 3]
+
+
+def test_stale_charts_in_scope_is_replaced() -> None:
+    """The reported symptom: a scope cache naming charts that are not present."""
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                "chartsInScope": [7, 17, 23],
+                "tabsInScope": ["TAB-gone"],
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1, 2, 3]
+    assert derived["native_filter_configuration"][0]["tabsInScope"] == ["TAB-1"]
+
+
+def test_scope_narrowed_to_a_tab() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["TAB-1"], "excluded": [3]},
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [2]
+    assert derived["native_filter_configuration"][0]["tabsInScope"] == ["TAB-1"]
+
+
+def test_dividers_and_unscoped_items() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {"id": "NATIVE_FILTER_DIVIDER-1", "chartsInScope": [7], "tabsInScope": []},
+            {"id": "DIVIDER-1", "type": "DIVIDER", "chartsInScope": [7]},
+            # A legacy chart customization targets a chart directly and only
+            # gains a scope once the client migrates it.
+            {"id": "CHART_CUSTOMIZATION-1", "chartId": 7, "chartsInScope": [7]},
+        ]
+    }
+
+    config = derive_scopes(metadata, POSITION_DATA, CHART_IDS)[
+        "native_filter_configuration"
+    ]
+
+    assert config[0]["chartsInScope"] == []
+    assert config[1]["chartsInScope"] == []
+    assert config[2]["chartsInScope"] == [7]
+
+
+def test_chart_configuration_drops_charts_not_on_the_dashboard() -> None:
+    metadata = {
+        "global_chart_configuration": {
+            "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            "chartsInScope": [7, 17],
+        },
+        "chart_configuration": {
+            "1": {"id": 1, "crossFilters": {"scope": "global", "chartsInScope": [17]}},
+            "2": {
+                "id": 2,
+                "crossFilters": {
+                    "scope": {"rootPath": ["TAB-1"], "excluded": []},
+                    "chartsInScope": [23],
+                },
+            },
+            "84": {"id": 84, "crossFilters": {"scope": "global", "chartsInScope": [7]}},
+        },
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["global_chart_configuration"]["chartsInScope"] == [1, 2, 3]
+    assert list(derived["chart_configuration"]) == ["1", "2"]
+    # A globally scoped chart emits to every other chart, never to itself.
+    assert derived["chart_configuration"]["1"]["crossFilters"]["chartsInScope"] == [
+        2,
+        3,
+    ]
+    assert derived["chart_configuration"]["2"]["crossFilters"]["chartsInScope"] == [
+        2,
+        3,
+    ]
+
+
+def test_selected_layers_target_their_chart_directly() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {
+                    "rootPath": ["TAB-1"],
+                    "excluded": [1],
+                    "selectedLayers": ["chart-1-layer-0"],
+                },
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    # Chart 1 is excluded and outside the rootPath, but a layer selection wins.
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1, 2, 3]
+
+
+def test_key_order_and_untouched_keys_are_kept() -> None:
+    metadata = {
+        "color_scheme": "supersetColors",
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "chartsInScope": [7],
+                "name": "Region",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            }
+        ],
+        "refresh_frequency": 0,
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert list(derived) == list(metadata)
+    assert list(derived["native_filter_configuration"][0]) == [
+        "id",
+        "chartsInScope",
+        "name",
+        "scope",
+        "tabsInScope",
+    ]
+    assert derived["color_scheme"] == "supersetColors"
+    assert derived["refresh_frequency"] == 0
+
+
+def test_derive_metadata_scopes_orders_by_dashboard_charts() -> None:
+    dashboard = SimpleNamespace(
+        position=POSITION_DATA,
+        slices=[SimpleNamespace(id=3), SimpleNamespace(id=1), SimpleNamespace(id=2)],
+    )
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            }
+        ]
+    }
+
+    derived = derive_metadata_scopes(dashboard, metadata)  # type: ignore[arg-type]
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [3, 1, 2]
+
+
+def test_derive_json_metadata_round_trip() -> None:
+    dashboard = SimpleNamespace(position=POSITION_DATA, slices=[SimpleNamespace(id=1)])
+    stored = json.dumps(
+        {
+            "native_filter_configuration": [
+                {
+                    "id": "NATIVE_FILTER-1",
+                    "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                    "chartsInScope": [61, 62],
+                }
+            ]
+        }
+    )
+
+    derived = json.loads(derive_json_metadata(dashboard, stored))  # type: ignore[arg-type]
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1]
+
+
+def test_derive_json_metadata_passes_through_unparsable_metadata() -> None:
+    dashboard = SimpleNamespace(position={}, slices=[])
+
+    assert derive_json_metadata(dashboard, "not json") == "not json"  # type: ignore[arg-type]
+    assert derive_json_metadata(dashboard, "[]") == "[]"  # type: ignore[arg-type]