From 7798c39c677ea030d7bddac047ec17e7069c3e4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 15:42:02 -0700 Subject: [PATCH] chore(deps): bump pandas from 2.1.4 to 2.3.3 (#42192) Signed-off-by: dependabot[bot] Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Amin Ghadersohi Co-authored-by: Claude Opus 4.8 (1M context) (cherry picked from commit 42a2aede78d0f279fe3ea2ea641f612da5c1af50) --- pyproject.toml | 2 +- requirements/base.txt | 4 +- requirements/development.txt | 6 +- superset/__init__.py | 14 +++- superset/utils/pandas_sqlalchemy_compat.py | 80 +++++++++++++++++++ .../utils/pandas_sqlalchemy_compat_test.py | 67 ++++++++++++++++ 6 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 superset/utils/pandas_sqlalchemy_compat.py create mode 100644 tests/unit_tests/utils/pandas_sqlalchemy_compat_test.py diff --git a/pyproject.toml b/pyproject.toml index ed80ae758f0..7f356e1b75f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ dependencies = [ "packaging", # -------------------------- # pandas and related (wanting pandas[performance] without numba as it's 100+MB and not needed) - "pandas[excel]>=2.1.4, <2.2", + "pandas[excel]>=2.3.3, <2.4", "bottleneck", # recommended performance dependency for pandas, see https://pandas.pydata.org/docs/getting_started/install.html#performance-dependencies-recommended # -------------------------- "parsedatetime", diff --git a/requirements/base.txt b/requirements/base.txt index c37afda03f8..477721105bb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -269,7 +269,7 @@ packaging==25.0 # limits # marshmallow # shillelagh -pandas==2.1.4 +pandas==2.3.3 # via apache-superset (pyproject.toml) paramiko==3.5.1 # via @@ -325,6 +325,8 @@ pyparsing==3.2.3 # via apache-superset (pyproject.toml) pysocks==1.7.1 # via urllib3 +python-calamine==0.8.2 + # via pandas python-dateutil==2.9.0.post0 # via # apache-superset (pyproject.toml) diff --git a/requirements/development.txt b/requirements/development.txt index de68a6427e0..a95dfef055f 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -628,7 +628,7 @@ packaging==25.0 # pytest # shillelagh # sqlalchemy-bigquery -pandas==2.1.4 +pandas==2.3.3 # via # -c requirements/base-constraint.txt # apache-superset @@ -810,6 +810,10 @@ pytest-mock==3.10.0 # via # apache-superset # apache-superset-extensions-cli +python-calamine==0.8.2 + # via + # -c requirements/base-constraint.txt + # pandas python-dateutil==2.9.0.post0 # via # -c requirements/base-constraint.txt diff --git a/superset/__init__.py b/superset/__init__.py index 91fcbf5ad8f..3b3c0c41f91 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -16,8 +16,18 @@ # under the License. from werkzeug.local import LocalProxy -from superset.app import create_app # noqa: F401 -from superset.extensions import ( +# pandas >= 2.2 advertises a minimum SQLAlchemy of 2.0 and silently ignores +# older installations, breaking DataFrame.to_sql / read_sql with SQLAlchemy +# 1.4 engines. Its SQL layer still works with 1.4, so restore support until +# Superset itself requires SQLAlchemy >= 2. Must run before any pandas SQL IO. +from superset.utils.pandas_sqlalchemy_compat import ( # noqa: E402 + restore_pandas_sqlalchemy_support, +) + +restore_pandas_sqlalchemy_support() + +from superset.app import create_app # noqa: E402, F401 +from superset.extensions import ( # noqa: E402 appbuilder, # noqa: F401 cache_manager, db, # noqa: F401 diff --git a/superset/utils/pandas_sqlalchemy_compat.py b/superset/utils/pandas_sqlalchemy_compat.py new file mode 100644 index 00000000000..3f3291418a7 --- /dev/null +++ b/superset/utils/pandas_sqlalchemy_compat.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Compatibility shim letting pandas >= 2.2 use SQLAlchemy 1.4 engines. + +pandas 2.2 raised its advertised minimum SQLAlchemy version to 2.0 as a +support-policy change. When an older SQLAlchemy is installed, pandas does not +fail loudly: ``pandas.io.sql`` silently pretends SQLAlchemy is absent, treats +Engine/Connection arguments as raw DBAPI connections, and falls back to its +sqlite-only code path, breaking every ``DataFrame.to_sql`` / ``read_sql`` +call site (dataset uploads, example data loading, annotation queries, filter +values). + +The pandas SQL layer itself still works with SQLAlchemy 1.4 because it only +uses the API subset common to SQLAlchemy 1.4 and 2.x. Lowering the advertised +minimum back to the pandas 2.1 value restores the working behavior. + +This module is obsolete once Superset requires SQLAlchemy >= 2; at that point +the patch becomes a no-op and the module (and its call site in +``superset/__init__.py``) can be deleted. +""" + +import logging + +import sqlalchemy +from packaging.version import Version + +logger = logging.getLogger(__name__) + +# The last pandas release line to support SQLAlchemy 1.4 (pandas 2.1) +# required at least this version. +_SQLALCHEMY_MINIMUM = "1.4.16" + + +def restore_pandas_sqlalchemy_support() -> None: + """Lower pandas' advertised SQLAlchemy minimum so 1.4 engines work. + + Only applies when the installed SQLAlchemy predates 2.0 and pandas + advertises a 2.x minimum; in every other combination this is a no-op. + Safe to call multiple times. + """ + if Version(sqlalchemy.__version__) >= Version("2.0.0"): + # pandas supports SQLAlchemy 2.x natively; nothing to patch. + return + + try: + from pandas.compat import _optional + except ImportError: + # The private module moved in a newer pandas; SQL IO with a pre-2.0 + # SQLAlchemy will misbehave, so make the situation diagnosable. + logger.warning( + "Could not adjust pandas' minimum SQLAlchemy version; " + "DataFrame.to_sql/read_sql may not accept SQLAlchemy %s engines", + sqlalchemy.__version__, + ) + return + + advertised = _optional.VERSIONS.get("sqlalchemy") + if advertised and Version(advertised) > Version(_SQLALCHEMY_MINIMUM): + _optional.VERSIONS["sqlalchemy"] = _SQLALCHEMY_MINIMUM + logger.debug( + "Lowered pandas' minimum SQLAlchemy version from %s to %s so " + "pandas SQL IO keeps working with the installed SQLAlchemy %s", + advertised, + _SQLALCHEMY_MINIMUM, + sqlalchemy.__version__, + ) diff --git a/tests/unit_tests/utils/pandas_sqlalchemy_compat_test.py b/tests/unit_tests/utils/pandas_sqlalchemy_compat_test.py new file mode 100644 index 00000000000..980f3123546 --- /dev/null +++ b/tests/unit_tests/utils/pandas_sqlalchemy_compat_test.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pandas as pd +from sqlalchemy import create_engine, types + +from superset.utils.pandas_sqlalchemy_compat import ( + restore_pandas_sqlalchemy_support, +) + + +def test_to_sql_accepts_sqlalchemy_engine_and_dtypes() -> None: + """ + ``DataFrame.to_sql`` must accept a SQLAlchemy engine plus SQLAlchemy + ``dtype`` objects regardless of the installed pandas/SQLAlchemy combo. + + This is the exact call shape used by dataset uploads + (``BaseEngineSpec.df_to_sql``), example data loading, and the test data + loaders; it breaks when pandas silently rejects the installed SQLAlchemy + as too old (pandas >= 2.2 with SQLAlchemy 1.x) and no compat shim is + applied. + """ + restore_pandas_sqlalchemy_support() + + engine = create_engine("sqlite://") + df = pd.DataFrame( + { + "name": ["a", "b"], + "num": [1, 2], + "ds": pd.to_datetime(["2021-01-01", "2021-01-02"]), + } + ) + df.to_sql( + "birth_names", + engine, + index=False, + dtype={"ds": types.DateTime(), "name": types.String(255)}, + method="multi", + chunksize=100, + ) + df.to_sql("birth_names", engine, index=False, if_exists="replace") + + result = pd.read_sql_query("SELECT name, num FROM birth_names", engine) + assert result["name"].tolist() == ["a", "b"] + assert result["num"].tolist() == [1, 2] + + +def test_restore_pandas_sqlalchemy_support_is_idempotent() -> None: + from pandas.compat import _optional + + restore_pandas_sqlalchemy_support() + first = _optional.VERSIONS.get("sqlalchemy") + restore_pandas_sqlalchemy_support() + assert _optional.VERSIONS.get("sqlalchemy") == first