fix(deps): add temporary SQLAlchemy 1.4 compatibility lane

Co-authored-by: Evan Rusackas <evan@preset.io>

.github/workflows/superset-python-unittest.yml

@@ -103,6 +103,48 @@ jobs:
           path: test-results/
           retention-days: 7
 
+  sqlalchemy14-compatibility:
+    needs: changes
+    if: needs.changes.outputs.python == 'true'
+    runs-on: ubuntu-26.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
+        with:
+          python-version: current
+      - name: Install SQLAlchemy 1.4 compatibility pair
+        # The development lock already supplies all transitive dependencies;
+        # replace only the correlated pair so the rest stays identical to SA2.
+        run: uv pip install --system --no-deps --reinstall -r requirements/sqlalchemy14.txt
+      - name: Validate legacy dependencies
+        run: uv pip check
+      - name: SQLAlchemy 1.4 compatibility tests
+        env:
+          SUPERSET_TESTENV: true
+          SUPERSET_SECRET_KEY: not-a-secret
+        run: |
+          pytest -q --cache-clear \
+            tests/unit_tests/initialization_test.py \
+            tests/unit_tests/extensions/test_sqlalchemy.py \
+            tests/unit_tests/commands/dataset/test_duplicate.py \
+            tests/unit_tests/commands/importers/v1/examples_test.py \
+            tests/unit_tests/databases/filters_test.py \
+            tests/unit_tests/db_engine_specs/test_duckdb.py \
+            tests/unit_tests/db_engine_specs/test_snowflake.py \
+            tests/unit_tests/db_engine_specs/test_trino.py \
+            tests/unit_tests/migrations \
+            tests/unit_tests/charts/commands/importers/v1/import_test.py \
+            tests/unit_tests/mcp_service/test_session_scope.py \
+            tests/unit_tests/mcp_service/test_auth_user_resolution.py \
+            tests/unit_tests/sql_lab_test.py \
+            tests/unit_tests/utils/test_core.py::test_pessimistic_connection_health_check_closes_transaction
+
   # Uploads the raw pull_request event payload so the "Python Unit Test
   # Results" workflow (running via workflow_run, in base-branch context) can
   # look up which PR/commit to annotate without checking out untrusted code.
@@ -124,7 +166,7 @@ jobs:
   # protection can require: it passes when unit-tests succeeded or was skipped,
   # and fails only on a real failure.
   unit-tests-required:
-    needs: [changes, unit-tests]
+    needs: [changes, unit-tests, sqlalchemy14-compatibility]
     if: always()
     runs-on: ubuntu-26.04
     timeout-minutes: 5
@@ -139,9 +181,15 @@ jobs:
       - name: Check unit-tests result
         env:
           RESULT: ${{ needs.unit-tests.result }}
+          LEGACY_RESULT: ${{ needs.sqlalchemy14-compatibility.result }}
         run: |
           if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
             echo "unit-tests did not pass (result: $RESULT)"
             exit 1
           fi
+          if [ "$LEGACY_RESULT" != "success" ] && [ "$LEGACY_RESULT" != "skipped" ]; then
+            echo "sqlalchemy14-compatibility did not pass (result: $LEGACY_RESULT)"
+            exit 1
+          fi
           echo "unit-tests result: $RESULT"
+          echo "sqlalchemy14-compatibility result: $LEGACY_RESULT"

docs/admin_docs/configuration/configuring-superset.mdx

@@ -97,6 +97,54 @@ for more information on how to configure it.
 
 At the very least, you'll want to change `SECRET_KEY` and `SQLALCHEMY_DATABASE_URI`. Continue reading for more about each of these.
 
+## Localizing D3 date and time labels
+
+`BABEL_DEFAULT_LOCALE` controls Superset's application translations, while
+`D3_TIME_FORMAT` provides localized date and time names to visualizations that
+use the D3 formatter registry, including Calendar Heatmap. Configure both when
+you want the application and chart labels to use the same locale.
+
+`D3_TIME_FORMAT` accepts partial overrides. For example, Russian month names
+can be configured in `superset_config.py` as follows:
+
+```python
+BABEL_DEFAULT_LOCALE = "ru"
+
+D3_TIME_FORMAT = {
+    "months": [
+        "Январь",
+        "Февраль",
+        "Март",
+        "Апрель",
+        "Май",
+        "Июнь",
+        "Июль",
+        "Август",
+        "Сентябрь",
+        "Октябрь",
+        "Ноябрь",
+        "Декабрь",
+    ],
+    "shortMonths": [
+        "Янв",
+        "Фев",
+        "Мар",
+        "Апр",
+        "Май",
+        "Июн",
+        "Июл",
+        "Авг",
+        "Сен",
+        "Окт",
+        "Ноя",
+        "Дек",
+    ],
+}
+```
+
+Restart Superset after changing `superset_config.py` so the frontend receives
+the updated formatter configuration.
+
 ## Chart-data query timing
 
 Set `CHART_DATA_INCLUDE_TIMING = True` to add an optional versioned timing object

pyproject.toml

@@ -60,11 +60,11 @@ dependencies = [
     "flask-login>=0.6.0, < 1.0",
     "flask-migrate>=4.1.0, <5.0",
     "flask-session>=0.4.0, <1.0",
-    # Bumped to 3.1.1 alongside the SQLAlchemy 2.0 core bump (discussion
-    # #40273, step 6), which resolves the session/app-context handling
-    # across Celery task boundaries that previously blocked this (see
-    # PR #42542).
-    "flask-sqlalchemy>=3.1.1, <4.0",
+    # The uncorrelated bounds permit two supported pairs: the default lock uses
+    # Flask-SQLAlchemy 3.1.1 with SQLAlchemy 2.x; the temporary downstream lane
+    # constrains Flask-SQLAlchemy 2.5.1 with SQLAlchemy 1.4.54. Consumers of the
+    # legacy lane must constrain both packages; see requirements/README.md.
+    "flask-sqlalchemy>=2.5.1, !=3.0.*, <4.0",
     "flask-wtf>=1.3.0, <2.0",
     "geopy",
     "greenlet<=3.5.4, >=3.5.4",
@@ -111,7 +111,7 @@ dependencies = [
     "sshtunnel>=0.4.0, <0.5",
     "simplejson>=4.1.1",
     "slack_sdk>=3.43.0, <4",
-    "sqlalchemy>=2.0.0, <2.1",
+    "sqlalchemy>=1.4.54, <2.1",
     "sqlalchemy-continuum>=1.6.0, <2.0.0",
     "sqlalchemy-utils>=0.42.1, <0.43", # expanding lowerbound to work with pydoris
     "sqlglot>=30.16.0, <31", # 30.16.0 adds Trino inline UDF IF/CASE routine statement parsing
@@ -161,8 +161,8 @@ datafusion = ["flightsql-dbapi>=0.2.2, <0.3"]
 db2 = ["ibm-db-sa<=0.4.4, >=0.4.4"]
 denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"]
 # sqlalchemy-dremio 3.0.5+ hard-pins sqlalchemy~=2.0.41, dropping 1.4.
-# Widened now that Superset's own SQLAlchemy 2.0 core bump has landed
-# (discussion #40273).
+# The supported default is SQLAlchemy 2. This extra is not available in the
+# temporary SQLAlchemy 1.4 compatibility lane (see requirements/README.md).
 dremio = ["sqlalchemy-dremio>=3.0.5, <4"]
 # <2 was an artificial ceiling; upstream has no SQLAlchemy version cap and
 # 1.1.10 already supports SQLAlchemy 2.0 (added `import_dbapi` in 1.1.7).
@@ -176,8 +176,7 @@ dynamodb = ["pydynamodb>=0.8.2"]
 solr = ["sqlalchemy-solr>=0.2.4.3"]
 elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"]
 # sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only
-# (>=6.0.0) with no dual-compat release. Bumped now that Superset's own
-# SQLAlchemy 2.0 core bump has landed (discussion #40273).
+# (>=6.0.0) with no dual-compat release. This extra is SQLAlchemy 2-only.
 exasol = ["sqlalchemy-exasol>=6.0.0, <8.0"]
 excel = ["xlrd>=2.0.2, <2.1"]
 # Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and
@@ -193,8 +192,7 @@ fastmcp = [
 ]
 # sqlalchemy-firebird >=2.0.0 unconditionally requires SQLAlchemy 2.0 on
 # Python >=3.8 (which covers Superset's >=3.11 floor), with no dual-compat
-# release. Bumped now that Superset's own SQLAlchemy 2.0 core bump has
-# landed (discussion #40273).
+# release. This extra is SQLAlchemy 2-only.
 firebird = ["sqlalchemy-firebird>=2.2.0"]
 firebolt = ["firebolt-sqlalchemy>=1.1.2, <2"]
 gevent = ["gevent>=26.7.0"]
@@ -234,13 +232,11 @@ presto = ["pyhive[presto]>=0.6.5"]
 trino = ["trino>=0.338.0"]
 prophet = ["prophet>=1.3.0, <2"]
 # sqlalchemy-redshift cuts hard from SQLAlchemy 1.4-only (0.8.x) to 2.0-only
-# (>=1.0.0) with no dual-compat release. Bumped now that Superset's own
-# SQLAlchemy 2.0 core bump has landed (discussion #40273).
+# (>=1.0.0) with no dual-compat release. This extra is SQLAlchemy 2-only.
 redshift = ["sqlalchemy-redshift>=1.0.0"]
 # No release of sqlalchemy-risingwave has ever supported both SQLAlchemy 1.4
 # and 2.0 (version numbers don't track SQLAlchemy compat monotonically).
-# Bumped to the 2.0-only line now that Superset's own SQLAlchemy 2.0 core
-# bump has landed (discussion #40273).
+# The selected driver line is SQLAlchemy 2-only.
 risingwave = ["sqlalchemy-risingwave>=2.0.0"]
 shillelagh = ["shillelagh[all]>=1.4.5, <2"]
 singlestore = ["sqlalchemy-singlestoredb>=1.2.1, <2"]

requirements/README.md

@@ -18,3 +18,29 @@ This will generate the pinned requirements in the `.txt` files, which will be us
 We recommend to everyone in the community to use the pinned requirements in their local development environments, to ensure consistency across different environments, though we don't force requirements as part of our python package semantics to allow flexibility for users to install different versions of the dependencies if they wish.
 
 Note that `development.txt` is a superset of what's in `base.txt`, and all version numbers for shared library should fully match at all times. `translations.txt` is meant as a supplemental file to be used in conjunction with the other requirements files, and is not meant to be used standalone.
+
+## Temporary SQLAlchemy 1.4 compatibility lane
+
+The generated `base.txt` and `development.txt` files remain the normal OSS
+environment and resolve SQLAlchemy 2.x with Flask-SQLAlchemy 3.1.1. A downstream
+that temporarily needs SQLAlchemy 1.4 must constrain **both** packages using
+`requirements/sqlalchemy14.txt` (SQLAlchemy 1.4.54 and Flask-SQLAlchemy 2.5.1).
+Constraining SQLAlchemy alone is intentionally unsupported because
+Flask-SQLAlchemy 3.1 requires SQLAlchemy 2.
+
+Python package metadata cannot express correlated alternatives such as “A 1.4
+with B 2.5, or A 2.x with B 3.1.” The published bounds therefore describe the
+union needed for downstream constraint files; they do not make arbitrary
+cross-pair combinations supported. CI tests the two exact pairs, and the OSS
+lock files prevent a default install from selecting the legacy pair.
+
+The core package and these commonly tested extras are compatible with both
+lanes: `bigquery`, `druid`, `duckdb`, `fastmcp`, `gevent`, `gsheets`, `mysql`,
+`postgres`, `presto`, `prophet`, `trino`, and `thumbnails`. The selected driver
+lines for `dremio`, `exasol`, `firebird`, `redshift`, and `risingwave` require
+SQLAlchemy 2 and must not be installed in the legacy lane. Other extras are not
+covered by the legacy CI lane and should be validated by downstream users.
+
+This lane is a temporary bridge for downstream migration, not a change to the
+OSS default. Remove the constraints, widened lower bounds, compatibility code,
+and legacy CI job together once those downstreams have moved to SQLAlchemy 2.

requirements/sqlalchemy14.txt

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+# Temporary SQLAlchemy 1.4 compatibility constraints. Always apply this file as
+# a unit: Flask-SQLAlchemy 3.1 requires SQLAlchemy 2 and is not a legacy option.
+SQLAlchemy==1.4.54
+Flask-SQLAlchemy==2.5.1

superset-core/pyproject.toml

@@ -45,7 +45,7 @@ dependencies = [
     "isodate>=0.7.0",
     "pyarrow>=16.0.0",
     "pydantic>=2.8.0",
-    "sqlalchemy>=2.0.0,<2.1",
+    "sqlalchemy>=1.4.54,<2.1",
     "sqlalchemy-utils>=0.38.0, <0.43", # expanding lowerbound to work with pydoris
     "sqlglot>=30.8.0, <31",
     "typing-extensions>=4.0.0",

superset-frontend/plugins/plugin-chart-calendar/src/Calendar.ts

@@ -22,7 +22,7 @@ import { getSequentialSchemeRegistry } from '@superset-ui/core';
 import { SupersetTheme } from '@apache-superset/core/theme';
 import { t } from '@apache-superset/core/translation';
 import CalHeatMapImport from './vendor/cal-heatmap';
-import { convertUTCTimestampToLocal } from './utils';
+import { convertUTCTimestampToLocal, getFormattedUTCTime } from './utils';
 
 // The vendor file is @ts-nocheck, so its export lacks type info.
 // Define a minimal constructor interface for use in this file.
@@ -103,6 +103,8 @@ function Calendar(element: HTMLElement, props: CalendarProps) {
   const subDomainTextFormat = showValues
     ? (_date: Date, value: number) => valueFormatter(value)
     : null;
+  const dateFormatter = (date: Date, format: string) =>
+    getFormattedUTCTime(date.getTime(), format);
 
   const metricsData = data.data;
 
@@ -166,6 +168,7 @@ function Calendar(element: HTMLElement, props: CalendarProps) {
       itemName: '',
       valueFormatter,
       timeFormatter,
+      dateFormatter,
       subDomainTextFormat,
     });
   });

superset-frontend/plugins/plugin-chart-calendar/src/vendor/cal-heatmap.ts

@@ -76,6 +76,8 @@ var CalHeatMap = function () {
 
     timeFormatter: d => d,
 
+    dateFormatter: null,
+
     domain: 'hour',
 
     subDomain: 'min',
@@ -1990,10 +1992,14 @@ CalHeatMap.prototype = {
 
     if (typeof format === 'function') {
       return format(d);
-    } else {
-      var f = d3.time.format(format);
-      return f(d);
     }
+
+    if (typeof this.options.dateFormatter === 'function') {
+      return this.options.dateFormatter(d, format);
+    }
+
+    var f = d3.time.format(format);
+    return f(d);
   },
 
   getSubDomainTitle: function (d) {

superset-frontend/plugins/plugin-chart-calendar/test/ReactCalendar.test.tsx

@@ -25,9 +25,11 @@ import {
   waitFor,
 } from 'spec/helpers/testing-library';
 import { CALENDAR_TOOLTIP_CLASS } from '../src/tooltip';
+import { convertUTCTimestampToLocal } from '../src/utils';
 
 interface MockCalHeatMapConfig {
   itemSelector: Element;
+  dateFormatter?: (date: Date, format: string) => string;
 }
 
 type MetricNameInput = string | string[];
@@ -38,6 +40,7 @@ let mockInitCallCount = 0;
 let mockThrowOnInitCall: number | null = null;
 let mockDestroyCallCount = 0;
 let mockDestroyedInstanceIds: string[] = [];
+let mockDateFormatter: MockCalHeatMapConfig['dateFormatter'];
 
 const mockTheme = {
   colorBgElevated: '#ffffff',
@@ -56,6 +59,7 @@ jest.mock('../src/vendor/cal-heatmap', () => ({
       } = require('../src/tooltip');
 
       mockInitCallCount += 1;
+      mockDateFormatter = config.dateFormatter;
       if (mockThrowOnInitCall === mockInitCallCount) {
         throw new Error('Mock CalHeatMap init failure');
       }
@@ -284,9 +288,28 @@ afterEach(() => {
   mockThrowOnInitCall = null;
   mockDestroyCallCount = 0;
   mockDestroyedInstanceIds = [];
+  mockDateFormatter = undefined;
   document.body.innerHTML = '';
 });
 
+test('Calendar provides a timezone-safe date formatter to CalHeatMap', () => {
+  const calendarOwner = document.createElement('div');
+  document.body.appendChild(calendarOwner);
+
+  Calendar(calendarOwner, {
+    ...createCalendarProps('localized-metric'),
+    theme: mockTheme,
+  });
+
+  if (!mockDateFormatter) {
+    throw new Error('Expected Calendar to configure a date formatter');
+  }
+
+  const localDate = new Date(convertUTCTimestampToLocal(Date.UTC(2024, 0, 1)));
+
+  expect(mockDateFormatter(localDate, '%Y-%m-%d')).toBe('2024-01-01');
+});
+
 test('rerender and unmount clean up only the affected calendar tooltips', () => {
   jest.useFakeTimers();
 

superset-frontend/plugins/plugin-chart-calendar/test/cal-heatmap.test.ts

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import CalHeatMapImport from '../src/vendor/cal-heatmap';
+
+type DateFormatter = (date: Date, format: string) => string;
+type FunctionalDateFormat = (date: Date) => string;
+
+interface CalHeatMapInstance {
+  options: {
+    dateFormatter: DateFormatter | null;
+  };
+  formatDate(date: Date, format: string | FunctionalDateFormat): string;
+}
+
+const CalHeatMap = CalHeatMapImport as unknown as new () => CalHeatMapInstance;
+
+test('CalHeatMap delegates string date formats to the configured formatter', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+  const dateFormatter = jest.fn<string, [Date, string]>(() => 'Январь');
+  calendar.options.dateFormatter = dateFormatter;
+
+  expect(calendar.formatDate(date, '%B')).toBe('Январь');
+  expect(dateFormatter).toHaveBeenCalledWith(date, '%B');
+});
+
+test('CalHeatMap preserves functional formatters over the configured formatter', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+  const dateFormatter = jest.fn<string, [Date, string]>(() => 'localized');
+  const functionalFormat = jest.fn<string, [Date]>(() => 'custom');
+  calendar.options.dateFormatter = dateFormatter;
+
+  expect(calendar.formatDate(date, functionalFormat)).toBe('custom');
+  expect(functionalFormat).toHaveBeenCalledWith(date);
+  expect(dateFormatter).not.toHaveBeenCalled();
+});
+
+test('CalHeatMap keeps the D3 formatter fallback', () => {
+  const calendar = new CalHeatMap();
+  const date = new Date(2024, 0, 1);
+
+  expect(calendar.formatDate(date, '%B')).toBe('January');
+});

superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx

@@ -867,7 +867,7 @@ function DatasourceEditor({
       return {
         ...metric,
         certification_details: certificationDetails || details,
-        warning_markdown: warningMarkdown || '',
+        warning_markdown: warningMarkdown || metric.warning_markdown || '',
         certified_by: certifiedBy || certifiedByMetric,
       };
     }),

superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditorMetricCertification.test.tsx

@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fetchMock from 'fetch-mock';
+import { screen, userEvent, waitFor } from 'spec/helpers/testing-library';
+import {
+  createProps,
+  DATASOURCE_ENDPOINT,
+  setupDatasourceEditorMocks,
+  cleanupAsyncOperations,
+  fastRender,
+  dismissDatasourceWarning,
+} from './DatasourceEditor.test.utils';
+
+beforeEach(() => {
+  fetchMock.get(DATASOURCE_ENDPOINT, [], { name: DATASOURCE_ENDPOINT });
+  setupDatasourceEditorMocks();
+});
+
+afterEach(async () => {
+  await cleanupAsyncOperations();
+  fetchMock.clearHistory().removeRoutes();
+});
+
+// Certifying a metric fills two adjacent fields in one visit to the expanded
+// row. Both are committed through TextControl's debounce, so the second one
+// used to land on the item as it looked before the first had been applied,
+// leaving the saved metric with details but no certifier.
+test('certifying a metric keeps both certified_by and certification_details', async () => {
+  const testProps = createProps();
+  fastRender(testProps);
+  await dismissDatasourceWarning();
+
+  await userEvent.click(await screen.findByTestId('collection-tab-Metrics'));
+  const expandToggles = await screen.findAllByLabelText(/expand row/i);
+  await userEvent.click(expandToggles[0]);
+
+  await userEvent.type(
+    await screen.findByPlaceholderText('Certified by'),
+    'Metric Certifier',
+  );
+  await userEvent.type(
+    await screen.findByPlaceholderText('Certification details'),
+    'Metric cert details',
+  );
+
+  await waitFor(() => {
+    const { calls } = testProps.onChange.mock;
+    const savedMetrics = calls[calls.length - 1]?.[0]?.metrics ?? [];
+    const saved = savedMetrics.find(metric => metric.metric_name === 'count');
+    expect(saved).toEqual(
+      expect.objectContaining({
+        certified_by: 'Metric Certifier',
+        certification_details: 'Metric cert details',
+      }),
+    );
+  });
+});

superset-frontend/src/components/Datasource/components/DatasourceEditor/tests/DatasourceEditorWarningMarkdown.test.tsx

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fetchMock from 'fetch-mock';
+import { screen, userEvent } from 'spec/helpers/testing-library';
+import {
+  createProps,
+  DATASOURCE_ENDPOINT,
+  setupDatasourceEditorMocks,
+  cleanupAsyncOperations,
+  fastRender,
+  dismissDatasourceWarning,
+} from './DatasourceEditor.test.utils';
+
+// Stub the Ace-backed control with a plain textarea. Ace spreads its document
+// across many spans and keeps only the keystroke buffer in its own textarea,
+// so asserting on the value the control receives is less brittle than
+// reaching into Ace's DOM.
+jest.mock('src/explore/components/controls/TextAreaControl', () => ({
+  __esModule: true,
+  default: ({
+    controlId,
+    value,
+    onChange,
+  }: {
+    controlId?: string;
+    value?: string;
+    onChange?: (value: string) => void;
+  }) => (
+    <textarea
+      data-test={`mock-textarea-${controlId}`}
+      value={value ?? ''}
+      onChange={event => onChange?.(event.target.value)}
+    />
+  ),
+}));
+
+beforeEach(() => {
+  fetchMock.get(DATASOURCE_ENDPOINT, [], { name: DATASOURCE_ENDPOINT });
+  setupDatasourceEditorMocks();
+});
+
+afterEach(async () => {
+  await cleanupAsyncOperations();
+  fetchMock.clearHistory().removeRoutes();
+});
+
+// Regression test for #42704. Explore's datasource payload (SqlMetric.data on
+// the backend) exposes warning_markdown as a flattened top-level field and
+// omits the raw `extra` JSON string that the /api/v1/dataset/{id} endpoint
+// backing the Datasets page provides. Deriving warning_markdown purely from
+// `extra` therefore dropped the saved text when the modal was opened from
+// Explore, leaving the Warning field blank on reopen.
+test('keeps a pre-existing top-level warning_markdown when the metric has no extra', async () => {
+  const baseProps = createProps();
+  const testProps = {
+    ...baseProps,
+    datasource: {
+      ...baseProps.datasource,
+      metrics: [
+        {
+          ...baseProps.datasource.metrics[0],
+          warning_markdown: 'existing warning',
+          extra: undefined,
+        },
+      ],
+    },
+  };
+
+  fastRender(testProps);
+  await dismissDatasourceWarning();
+
+  const metricsTab = await screen.findByTestId('collection-tab-Metrics');
+  await userEvent.click(metricsTab);
+
+  const expandToggles = await screen.findAllByLabelText(/expand row/i);
+  await userEvent.click(expandToggles[0]);
+
+  expect(
+    await screen.findByTestId('mock-textarea-warning_markdown'),
+  ).toHaveValue('existing warning');
+});

superset-frontend/src/components/Datasource/components/Fieldset/index.tsx

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { ReactNode, useCallback } from 'react';
+import { ReactNode, useCallback, useEffect, useRef } from 'react';
 import { Divider, Form, Typography } from '@superset-ui/core/components';
 import { css } from '@apache-superset/core/theme';
 import { recurseReactClone } from '../../utils';
@@ -39,14 +39,24 @@ export default function Fieldset({
   title = null,
   compact = false,
 }: FieldsetProps) {
+  // Controls report their edits asynchronously - TextControl debounces by
+  // FAST_DEBOUNCE - so the callback that eventually fires was built during an
+  // earlier render. Spreading that render's `item` rebuilds the whole record
+  // from a snapshot taken before a sibling field committed, dropping the value
+  // the user typed first. Reading off a ref merges into the latest commit.
+  const itemRef = useRef(item);
+  useEffect(() => {
+    itemRef.current = item;
+  }, [item]);
+
   const handleChange = useCallback(
     (fieldKey: fieldKeyType, val: any) => {
       onChange?.({
-        ...item,
+        ...itemRef.current,
         [fieldKey]: val,
       });
     },
-    [onChange, item],
+    [onChange],
   );
 
   const propExtender = (field: { props: { fieldKey: fieldKeyType } }) => ({

superset-frontend/src/dashboard/actions/nativeFilters.ts

@@ -27,7 +27,7 @@ import {
 } from '@superset-ui/core';
 import { Dispatch } from 'redux';
 import { RootState } from 'src/dashboard/types';
-import { cloneDeep } from 'lodash-es';
+import { cloneDeep, omit } from 'lodash-es';
 import { setDataMaskForFilterChangesComplete } from 'src/dataMask/actions';
 import { HYDRATE_DASHBOARD } from './hydrate';
 import {
@@ -90,12 +90,20 @@ export const setFilterConfiguration =
     });
     try {
       const response = await updateFilters(filterChanges);
+      // chartsInScope/tabsInScope are derived from the live layout, and the
+      // response carries the persisted copy for every filter - including the
+      // ones this save never touched, whose copy is whatever was stored when
+      // the dashboard was last saved. Dropping them lets the reducers keep the
+      // scopes calculateScopes already computed for this session.
+      const savedFilters = response.result.map(
+        filter => omit(filter, ['chartsInScope', 'tabsInScope']) as Filter,
+      );
       dispatch({
         type: SET_NATIVE_FILTERS_CONFIG_COMPLETE,
-        filterChanges: response.result,
+        filterChanges: savedFilters,
         deletedIds: filterChanges.deleted,
       });
-      dispatch(nativeFiltersConfigChanged(response.result));
+      dispatch(nativeFiltersConfigChanged(savedFilters));
       dispatch(setDataMaskForFilterChangesComplete(filterChanges, oldFilters));
     } catch (err) {
       dispatch({

superset/daos/dashboard.py

@@ -34,6 +34,7 @@ from superset.commands.dashboard.exceptions import (
     DashboardUpdateFailedError,
 )
 from superset.daos.base import BaseDAO, ColumnOperator, ColumnOperatorEnum
+from superset.dashboards.filter_scope import derive_metadata_scopes
 from superset.dashboards.filters import DashboardAccessFilter
 from superset.exceptions import SupersetSecurityException
 from superset.extensions import db
@@ -547,7 +548,9 @@ class DashboardDAO(BaseDAO[Dashboard]):
         cls, id: str
     ) -> dict[str, list[dict[str, Any]]]:
         dashboard = cls.get_by_id_or_slug(id)
-        metadata = json.loads(dashboard.json_metadata or "{}")
+        metadata = derive_metadata_scopes(
+            dashboard, json.loads(dashboard.json_metadata or "{}")
+        )
         native_filter_configuration = metadata.get("native_filter_configuration", [])
 
         tab_filters = defaultdict(list)
@@ -617,6 +620,13 @@ class DashboardDAO(BaseDAO[Dashboard]):
             metadata["native_filter_configuration"] = updated_configuration
             dashboard.json_metadata = json.dumps(metadata)
 
+            # The client rebuilds its in-scope state from this response, so hand
+            # back derived scopes rather than the stored caches, which are stale
+            # for every filter the caller did not touch.
+            updated_configuration = derive_metadata_scopes(dashboard, metadata)[
+                "native_filter_configuration"
+            ]
+
         return updated_configuration
 
     @classmethod

superset/dashboards/api.py

@@ -91,6 +91,7 @@ from superset.commands.importers.v1.utils import get_contents_from_bundle
 from superset.commands.purge import PurgeArchivedCommand, SoftDeleteBinding
 from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod
 from superset.daos.dashboard import DashboardDAO, EmbeddedDashboardDAO
+from superset.dashboards.filter_scope import derive_json_metadata
 from superset.dashboards.filters import (
     DashboardAccessFilter,
     DashboardCertifiedFilter,
@@ -653,6 +654,12 @@ class DashboardRestApi(
             schema = self.dashboard_get_response_schema
 
         result = schema.dump(dash)
+        if json_metadata := result.get("json_metadata"):
+            # The stored scope caches (``chartsInScope``, ``tabsInScope``,
+            # ``chart_configuration``) go stale as soon as the layout changes;
+            # derive them so callers see the same document the dashboard client
+            # computes for itself.
+            result["json_metadata"] = derive_json_metadata(dash, json_metadata)
         if "charts" in result:
             # Only name the member charts the caller can access, consistent with
             # the per-object narrowing applied to the dashboard's datasets and

superset/dashboards/filter_scope.py

@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Derive filter-scope caches in ``json_metadata`` from the dashboard layout.
+
+``chartsInScope`` / ``tabsInScope`` on a native filter, and the ``chartsInScope``
+lists inside ``chart_configuration`` / ``global_chart_configuration``, are
+denormalized caches of the authoritative ``scope`` plus ``position_json``. They
+are written when a dashboard is saved and are never revisited afterwards, so a
+dashboard that has charts added or removed - or that was seeded, exported or
+imported - carries scope arrays naming charts it does not contain.
+
+The dashboard client already ignores the stored values and recomputes them from
+the live layout on every load, which is why the JSON Metadata panel and
+``GET /api/v1/dashboard/{id}`` disagreed on a dashboard nobody had ever saved.
+Deriving them on read makes the API agree with the client and keeps integrations
+that read ``native_filter_configuration`` from receiving dangling chart ids.
+
+The rules mirror the client (``superset-frontend/src/dashboard/util``):
+``calculateScopes``, ``getChartIdsInFilterScope``, ``findTabsWithChartsInScope``
+and ``getCrossFiltersConfiguration``.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, TYPE_CHECKING
+
+from superset.utils import json
+
+if TYPE_CHECKING:
+    from superset.models.dashboard import Dashboard
+
+CHART_TYPE = "CHART"
+TAB_TYPE = "TAB"
+NATIVE_FILTER_DIVIDER_PREFIX = "NATIVE_FILTER_DIVIDER-"
+DIVIDER_TYPES = frozenset({"DIVIDER", "CHART_CUSTOMIZATION_DIVIDER"})
+
+# ``chart-<chartId>-layer-<layerIndex>``, the per-layer scope keys a deck.gl
+# multi-layer chart contributes to ``scope.selectedLayers``.
+LAYER_SELECTION_RE = re.compile(r"^chart-(\d+)-layer-(\d+)$")
+
+ChartLayoutItems = dict[int, list[dict[str, Any]]]
+
+
+def build_chart_layout_items(position_data: dict[str, Any]) -> ChartLayoutItems:
+    """Map each chart id in the layout to the layout items that render it."""
+    chart_layout_items: ChartLayoutItems = {}
+    for item in position_data.values():
+        if not isinstance(item, dict) or item.get("type") != CHART_TYPE:
+            continue
+        chart_id = item.get("meta", {}).get("chartId")
+        if isinstance(chart_id, int):
+            chart_layout_items.setdefault(chart_id, []).append(item)
+    return chart_layout_items
+
+
+def get_chart_ids_in_scope(
+    scope: dict[str, Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[int]:
+    """Charts covered by ``scope``, in ``chart_ids`` order."""
+    excluded = set(scope.get("excluded") or [])
+    root_path = set(scope.get("rootPath") or [])
+
+    def in_scope(chart_id: int) -> bool:
+        if chart_id in excluded:
+            return False
+        return any(
+            parent in root_path
+            for layout_item in chart_layout_items.get(chart_id, [])
+            for parent in layout_item.get("parents") or []
+        )
+
+    selected_layers = scope.get("selectedLayers") or []
+    if not selected_layers:
+        return [chart_id for chart_id in chart_ids if in_scope(chart_id)]
+
+    # A layer selection targets its chart directly, and suppresses the
+    # rootPath/excluded test for that chart.
+    charts_with_layer_selections = set()
+    targeted: list[int] = []
+    chart_id_set = set(chart_ids)
+    for selection_key in selected_layers:
+        if match := LAYER_SELECTION_RE.match(str(selection_key)):
+            chart_id = int(match.group(1))
+            charts_with_layer_selections.add(chart_id)
+            if chart_id in chart_id_set and chart_id not in targeted:
+                targeted.append(chart_id)
+
+    return targeted + [
+        chart_id
+        for chart_id in chart_ids
+        if chart_id not in charts_with_layer_selections
+        and chart_id not in targeted
+        and in_scope(chart_id)
+    ]
+
+
+def get_tabs_in_scope(
+    charts_in_scope: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[str]:
+    """Tabs holding at least one of ``charts_in_scope``."""
+    tabs_in_scope: list[str] = []
+    seen: set[str] = set()
+    for chart_id in charts_in_scope:
+        for layout_item in chart_layout_items.get(chart_id, []):
+            for parent in layout_item.get("parents") or []:
+                if parent.startswith(f"{TAB_TYPE}-") and parent not in seen:
+                    seen.add(parent)
+                    tabs_in_scope.append(parent)
+    return tabs_in_scope
+
+
+def _is_divider(item: dict[str, Any]) -> bool:
+    return (
+        str(item.get("id", "")).startswith(NATIVE_FILTER_DIVIDER_PREFIX)
+        or item.get("type") in DIVIDER_TYPES
+    )
+
+
+def _derive_item_scopes(
+    items: list[Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> list[Any]:
+    """Restamp ``chartsInScope`` / ``tabsInScope`` on scoped config items.
+
+    Items without a usable ``scope`` are returned untouched: legacy chart
+    customizations target a chart directly and only gain a ``scope`` once the
+    client migrates them, so overwriting their cache here would drop targeting
+    the client still needs.
+    """
+    derived = []
+    for item in items:
+        if not isinstance(item, dict):
+            derived.append(item)
+            continue
+        if _is_divider(item):
+            derived.append({**item, "chartsInScope": [], "tabsInScope": []})
+            continue
+        scope = item.get("scope")
+        if not isinstance(scope, dict) or not isinstance(scope.get("excluded"), list):
+            derived.append(item)
+            continue
+        charts_in_scope = get_chart_ids_in_scope(scope, chart_ids, chart_layout_items)
+        derived.append(
+            {
+                **item,
+                "chartsInScope": charts_in_scope,
+                "tabsInScope": get_tabs_in_scope(charts_in_scope, chart_layout_items),
+            }
+        )
+    return derived
+
+
+def _derive_cross_filter_scopes(
+    metadata: dict[str, Any],
+    chart_ids: list[int],
+    chart_layout_items: ChartLayoutItems,
+) -> None:
+    global_config = metadata.get("global_chart_configuration")
+    global_charts_in_scope = chart_ids
+    if isinstance(global_config, dict) and isinstance(global_config.get("scope"), dict):
+        global_charts_in_scope = get_chart_ids_in_scope(
+            global_config["scope"], chart_ids, chart_layout_items
+        )
+        metadata["global_chart_configuration"] = {
+            **global_config,
+            "chartsInScope": global_charts_in_scope,
+        }
+
+    chart_configuration = metadata.get("chart_configuration")
+    if not isinstance(chart_configuration, dict):
+        return
+
+    derived_configuration = {}
+    for key, config in chart_configuration.items():
+        try:
+            chart_id = int(key)
+        except (TypeError, ValueError):
+            derived_configuration[key] = config
+            continue
+        # Config for a chart no longer on the dashboard is dead weight; the
+        # client drops it on load for the same reason.
+        if chart_id not in chart_layout_items:
+            continue
+        if not isinstance(config, dict):
+            derived_configuration[key] = config
+            continue
+        cross_filters = config.get("crossFilters")
+        if not isinstance(cross_filters, dict):
+            derived_configuration[key] = config
+            continue
+        scope = cross_filters.get("scope")
+        if isinstance(scope, dict):
+            charts_in_scope = get_chart_ids_in_scope(
+                scope, chart_ids, chart_layout_items
+            )
+        else:
+            # Anything that is not an explicit scope object points at the
+            # dashboard-wide scope, which never includes the emitting chart.
+            charts_in_scope = [cid for cid in global_charts_in_scope if cid != chart_id]
+        derived_configuration[key] = {
+            **config,
+            "crossFilters": {**cross_filters, "chartsInScope": charts_in_scope},
+        }
+
+    metadata["chart_configuration"] = derived_configuration
+
+
+def derive_scopes(
+    metadata: dict[str, Any],
+    position_data: dict[str, Any],
+    chart_ids: list[int],
+) -> dict[str, Any]:
+    """Return ``metadata`` with every derived scope cache recomputed.
+
+    ``chart_ids`` orders the resulting ``chartsInScope`` arrays and should be the
+    dashboard's chart ids as the client sees them, so that the API and the JSON
+    Metadata panel produce byte-identical documents.
+    """
+    derived = dict(metadata)
+    chart_layout_items = build_chart_layout_items(position_data)
+
+    for key in ("native_filter_configuration", "chart_customization_config"):
+        config = derived.get(key)
+        if isinstance(config, list):
+            derived[key] = _derive_item_scopes(config, chart_ids, chart_layout_items)
+
+    _derive_cross_filter_scopes(derived, chart_ids, chart_layout_items)
+    return derived
+
+
+def derive_metadata_scopes(
+    dashboard: Dashboard,
+    metadata: dict[str, Any],
+) -> dict[str, Any]:
+    """``derive_scopes`` for a dashboard model's parsed ``json_metadata``."""
+    return derive_scopes(
+        metadata,
+        dashboard.position,
+        [slc.id for slc in dashboard.slices],
+    )
+
+
+def derive_json_metadata(dashboard: Dashboard, json_metadata: str) -> str:
+    """``derive_metadata_scopes`` over a raw ``json_metadata`` string.
+
+    Metadata that does not parse as a JSON object is handed back untouched -
+    reading a dashboard is not the place to start rejecting documents that have
+    always been served as-is.
+    """
+    try:
+        metadata = json.loads(json_metadata)
+    except (TypeError, ValueError):
+        return json_metadata
+    if not isinstance(metadata, dict):
+        return json_metadata
+    return json.dumps(derive_metadata_scopes(dashboard, metadata))

superset/utils/core.py

@@ -816,7 +816,8 @@ def pessimistic_connection_handling(some_engine: Engine) -> None:
             # the SELECT of a scalar value without a table is
             # appropriately formatted for the backend
             connection.scalar(select(1))
-            connection.rollback()  # pylint: disable=consider-using-transaction
+            if transaction := connection.get_transaction():
+                transaction.rollback()
         except exc.DBAPIError as err:
             # catch SQLAlchemy's DBAPIError, which is a wrapper
             # for the DBAPI's exception.  It includes a .connection_invalidated
@@ -829,7 +830,8 @@ def pessimistic_connection_handling(some_engine: Engine) -> None:
                 # here also causes the whole connection pool to be invalidated
                 # so that all stale connections are discarded.
                 connection.scalar(select(1))
-                connection.rollback()  # pylint: disable=consider-using-transaction
+                if transaction := connection.get_transaction():
+                    transaction.rollback()
             else:
                 raise
         finally:

tests/integration_tests/dashboards/api_tests.py

@@ -646,6 +646,65 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
         db.session.delete(dashboard)
         db.session.commit()
 
+    def test_get_dashboard_derives_stale_filter_scope(self):
+        """
+        Dashboard API: ``chartsInScope`` is derived from the layout, not read
+        back from the stored cache (sc-116923).
+        """
+        admin = self.get_user("admin")
+        slices = db.session.query(Slice).limit(2).all()
+        positions = {
+            "ROOT_ID": {"id": "ROOT_ID", "type": "ROOT", "children": ["GRID_ID"]},
+            "GRID_ID": {"id": "GRID_ID", "type": "GRID", "parents": ["ROOT_ID"]},
+        }
+        for slc in slices:
+            positions[f"CHART-{slc.id}"] = {
+                "id": f"CHART-{slc.id}",
+                "type": "CHART",
+                "meta": {"chartId": slc.id},
+                "parents": ["ROOT_ID", "GRID_ID"],
+            }
+        # A scope naming charts the dashboard does not contain - the state every
+        # seeded and imported dashboard starts in.
+        stored_metadata = {
+            "native_filter_configuration": [
+                {
+                    "id": "NATIVE_FILTER-1",
+                    "name": "Region",
+                    "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                    "chartsInScope": [90001, 90002],
+                    "tabsInScope": ["TAB-gone"],
+                }
+            ]
+        }
+        dashboard = self.insert_dashboard(
+            "scope-cache",
+            "scope-cache",
+            [admin.id],
+            slices=slices,
+            position_json=json.dumps(positions),
+            json_metadata=json.dumps(stored_metadata),
+        )
+
+        self.login(ADMIN_USERNAME)
+        rv = self.get_assert_metric(f"api/v1/dashboard/{dashboard.id}", "get")
+        assert rv.status_code == 200
+
+        response_metadata = json.loads(
+            json.loads(rv.data.decode("utf-8"))["result"]["json_metadata"]
+        )
+        native_filter = response_metadata["native_filter_configuration"][0]
+        assert sorted(native_filter["chartsInScope"]) == sorted(
+            slc.id for slc in slices
+        )
+        assert native_filter["tabsInScope"] == []
+        assert native_filter["name"] == "Region"
+        # Deriving is read-only; the stored document is left alone.
+        assert json.loads(dashboard.json_metadata) == stored_metadata
+
+        db.session.delete(dashboard)
+        db.session.commit()
+
     def test_get_dashboard_with_columns(self):
         """
         Dashboard API: Test get dashboard with column selection via q param

tests/unit_tests/dashboards/filter_scope_test.py

@@ -0,0 +1,233 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from types import SimpleNamespace
+from typing import Any
+
+from superset.dashboards.filter_scope import (
+    derive_json_metadata,
+    derive_metadata_scopes,
+    derive_scopes,
+)
+from superset.utils import json
+
+# Two charts in a tab, one chart outside it.
+POSITION_DATA: dict[str, Any] = {
+    "ROOT_ID": {"id": "ROOT_ID", "type": "ROOT", "children": ["GRID_ID"]},
+    "CHART-outside": {
+        "id": "CHART-outside",
+        "type": "CHART",
+        "meta": {"chartId": 1},
+        "parents": ["ROOT_ID", "GRID_ID"],
+    },
+    "CHART-in-tab": {
+        "id": "CHART-in-tab",
+        "type": "CHART",
+        "meta": {"chartId": 2},
+        "parents": ["ROOT_ID", "GRID_ID", "TABS-1", "TAB-1"],
+    },
+    "CHART-also-in-tab": {
+        "id": "CHART-also-in-tab",
+        "type": "CHART",
+        "meta": {"chartId": 3},
+        "parents": ["ROOT_ID", "GRID_ID", "TABS-1", "TAB-1"],
+    },
+    "MARKDOWN-1": {"id": "MARKDOWN-1", "type": "MARKDOWN", "parents": ["ROOT_ID"]},
+}
+CHART_IDS = [1, 2, 3]
+
+
+def test_stale_charts_in_scope_is_replaced() -> None:
+    """The reported symptom: a scope cache naming charts that are not present."""
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                "chartsInScope": [7, 17, 23],
+                "tabsInScope": ["TAB-gone"],
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1, 2, 3]
+    assert derived["native_filter_configuration"][0]["tabsInScope"] == ["TAB-1"]
+
+
+def test_scope_narrowed_to_a_tab() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["TAB-1"], "excluded": [3]},
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [2]
+    assert derived["native_filter_configuration"][0]["tabsInScope"] == ["TAB-1"]
+
+
+def test_dividers_and_unscoped_items() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {"id": "NATIVE_FILTER_DIVIDER-1", "chartsInScope": [7], "tabsInScope": []},
+            {"id": "DIVIDER-1", "type": "DIVIDER", "chartsInScope": [7]},
+            # A legacy chart customization targets a chart directly and only
+            # gains a scope once the client migrates it.
+            {"id": "CHART_CUSTOMIZATION-1", "chartId": 7, "chartsInScope": [7]},
+        ]
+    }
+
+    config = derive_scopes(metadata, POSITION_DATA, CHART_IDS)[
+        "native_filter_configuration"
+    ]
+
+    assert config[0]["chartsInScope"] == []
+    assert config[1]["chartsInScope"] == []
+    assert config[2]["chartsInScope"] == [7]
+
+
+def test_chart_configuration_drops_charts_not_on_the_dashboard() -> None:
+    metadata = {
+        "global_chart_configuration": {
+            "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            "chartsInScope": [7, 17],
+        },
+        "chart_configuration": {
+            "1": {"id": 1, "crossFilters": {"scope": "global", "chartsInScope": [17]}},
+            "2": {
+                "id": 2,
+                "crossFilters": {
+                    "scope": {"rootPath": ["TAB-1"], "excluded": []},
+                    "chartsInScope": [23],
+                },
+            },
+            "84": {"id": 84, "crossFilters": {"scope": "global", "chartsInScope": [7]}},
+        },
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert derived["global_chart_configuration"]["chartsInScope"] == [1, 2, 3]
+    assert list(derived["chart_configuration"]) == ["1", "2"]
+    # A globally scoped chart emits to every other chart, never to itself.
+    assert derived["chart_configuration"]["1"]["crossFilters"]["chartsInScope"] == [
+        2,
+        3,
+    ]
+    assert derived["chart_configuration"]["2"]["crossFilters"]["chartsInScope"] == [
+        2,
+        3,
+    ]
+
+
+def test_selected_layers_target_their_chart_directly() -> None:
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {
+                    "rootPath": ["TAB-1"],
+                    "excluded": [1],
+                    "selectedLayers": ["chart-1-layer-0"],
+                },
+            }
+        ]
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    # Chart 1 is excluded and outside the rootPath, but a layer selection wins.
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1, 2, 3]
+
+
+def test_key_order_and_untouched_keys_are_kept() -> None:
+    metadata = {
+        "color_scheme": "supersetColors",
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "chartsInScope": [7],
+                "name": "Region",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            }
+        ],
+        "refresh_frequency": 0,
+    }
+
+    derived = derive_scopes(metadata, POSITION_DATA, CHART_IDS)
+
+    assert list(derived) == list(metadata)
+    assert list(derived["native_filter_configuration"][0]) == [
+        "id",
+        "chartsInScope",
+        "name",
+        "scope",
+        "tabsInScope",
+    ]
+    assert derived["color_scheme"] == "supersetColors"
+    assert derived["refresh_frequency"] == 0
+
+
+def test_derive_metadata_scopes_orders_by_dashboard_charts() -> None:
+    dashboard = SimpleNamespace(
+        position=POSITION_DATA,
+        slices=[SimpleNamespace(id=3), SimpleNamespace(id=1), SimpleNamespace(id=2)],
+    )
+    metadata = {
+        "native_filter_configuration": [
+            {
+                "id": "NATIVE_FILTER-1",
+                "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+            }
+        ]
+    }
+
+    derived = derive_metadata_scopes(dashboard, metadata)  # type: ignore[arg-type]
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [3, 1, 2]
+
+
+def test_derive_json_metadata_round_trip() -> None:
+    dashboard = SimpleNamespace(position=POSITION_DATA, slices=[SimpleNamespace(id=1)])
+    stored = json.dumps(
+        {
+            "native_filter_configuration": [
+                {
+                    "id": "NATIVE_FILTER-1",
+                    "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                    "chartsInScope": [61, 62],
+                }
+            ]
+        }
+    )
+
+    derived = json.loads(derive_json_metadata(dashboard, stored))  # type: ignore[arg-type]
+
+    assert derived["native_filter_configuration"][0]["chartsInScope"] == [1]
+
+
+def test_derive_json_metadata_passes_through_unparsable_metadata() -> None:
+    dashboard = SimpleNamespace(position={}, slices=[])
+
+    assert derive_json_metadata(dashboard, "not json") == "not json"  # type: ignore[arg-type]
+    assert derive_json_metadata(dashboard, "[]") == "[]"  # type: ignore[arg-type]

tests/unit_tests/databases/filters_test.py

@@ -117,7 +117,9 @@ def test_database_filter(mocker: MockerFixture) -> None:
         engine,
         compile_kwargs={"literal_binds": True},
     )
-    assert (
-        str(compiled_query)
-        == "SELECT dbs.id, dbs.verbose_name, dbs.database_name, dbs.sqlalchemy_uri, dbs.password, dbs.cache_timeout, dbs.select_as_create_table_as, dbs.expose_in_sqllab, dbs.configuration_method, dbs.allow_run_async, dbs.allow_file_upload, dbs.allow_ctas, dbs.allow_cvas, dbs.allow_dml, dbs.force_ctas_schema, dbs.extra, dbs.encrypted_extra, dbs.impersonate_user, dbs.server_cert, dbs.is_managed_externally, dbs.external_url, dbs.created_on, dbs.changed_on, dbs.created_by_fk, dbs.changed_by_fk, dbs.uuid, ssh_tunnels_1.id AS id_1, ssh_tunnels_1.database_id, ssh_tunnels_1.server_address, ssh_tunnels_1.server_port, ssh_tunnels_1.username, ssh_tunnels_1.password AS password_1, ssh_tunnels_1.private_key, ssh_tunnels_1.private_key_password, ssh_tunnels_1.server_host_key, ssh_tunnels_1.created_on AS created_on_1, ssh_tunnels_1.changed_on AS changed_on_1, ssh_tunnels_1.created_by_fk AS created_by_fk_1, ssh_tunnels_1.changed_by_fk AS changed_by_fk_1, ssh_tunnels_1.extra_json, ssh_tunnels_1.uuid AS uuid_1 \nFROM dbs LEFT OUTER JOIN ssh_tunnels AS ssh_tunnels_1 ON dbs.id = ssh_tunnels_1.database_id \nWHERE ('[' || dbs.database_name || '].(id:' || CAST(dbs.id AS VARCHAR) || ')') IN ('[my_db].(id:42)', '[my_other_db].(id:43)') OR dbs.database_name IN ('my_db', 'my_other_db', 'third_db')"  # noqa: E501
-    )
+    sql = str(compiled_query)
+    # SQLAlchemy 1.4 and 2.x produce the same filter but differ in inherited
+    # column order and optional grouping parentheses when compiling it.
+    assert "FROM dbs LEFT OUTER JOIN ssh_tunnels" in sql
+    assert "'[my_db].(id:42)', '[my_other_db].(id:43)'" in sql
+    assert "dbs.database_name IN ('my_db', 'my_other_db', 'third_db')" in sql

tests/unit_tests/utils/test_core.py

@@ -25,6 +25,7 @@ import pytest
 from flask import current_app
 from pandas.api.types import is_datetime64_dtype
 from pytest_mock import MockerFixture
+from sqlalchemy import create_engine
 
 from superset.exceptions import SupersetException
 from superset.utils.core import (
@@ -49,6 +50,7 @@ from superset.utils.core import (
     normalize_dttm_col,
     parse_boolean_string,
     parse_js_uri_path_item,
+    pessimistic_connection_handling,
     QueryObjectFilterClause,
     QuerySource,
     remove_extra_adhoc_filters,
@@ -2088,3 +2090,11 @@ def test_extract_dataframe_dtypes_with_duplicate_columns() -> None:
     df = pd.DataFrame([[1, 2, 3]], columns=["a", "b", "a"])
     result = extract_dataframe_dtypes(df)
     assert len(result) == 3
+
+
+def test_pessimistic_connection_health_check_closes_transaction() -> None:
+    engine = create_engine("sqlite://")
+    pessimistic_connection_handling(engine)
+
+    with engine.connect() as connection:
+        assert not connection.in_transaction()