chore(deps): bump pandas from 2.1.4 to 2.3.3 (#42192)

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Amin Ghadersohi <amin.ghadersohi@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
(cherry picked from commit 42a2aede78)

pyproject.toml

@@ -76,7 +76,7 @@ dependencies = [
     "packaging",
     # --------------------------
     # pandas and related (wanting pandas[performance] without numba as it's 100+MB and not needed)
-    "pandas[excel]>=2.1.4, <2.2",
+    "pandas[excel]>=2.3.3, <2.4",
     "bottleneck", # recommended performance dependency for pandas, see https://pandas.pydata.org/docs/getting_started/install.html#performance-dependencies-recommended
     # --------------------------
     "parsedatetime",

requirements/base.txt

@@ -269,7 +269,7 @@ packaging==25.0
     #   limits
     #   marshmallow
     #   shillelagh
-pandas==2.1.4
+pandas==2.3.3
     # via apache-superset (pyproject.toml)
 paramiko==3.5.1
     # via
@@ -325,6 +325,8 @@ pyparsing==3.2.3
     # via apache-superset (pyproject.toml)
 pysocks==1.7.1
     # via urllib3
+python-calamine==0.8.2
+    # via pandas
 python-dateutil==2.9.0.post0
     # via
     #   apache-superset (pyproject.toml)

requirements/development.txt

@@ -628,7 +628,7 @@ packaging==25.0
     #   pytest
     #   shillelagh
     #   sqlalchemy-bigquery
-pandas==2.1.4
+pandas==2.3.3
     # via
     #   -c requirements/base-constraint.txt
     #   apache-superset
@@ -810,6 +810,10 @@ pytest-mock==3.10.0
     # via
     #   apache-superset
     #   apache-superset-extensions-cli
+python-calamine==0.8.2
+    # via
+    #   -c requirements/base-constraint.txt
+    #   pandas
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/base-constraint.txt

superset/__init__.py

@@ -16,8 +16,18 @@
 # under the License.
 from werkzeug.local import LocalProxy
 
-from superset.app import create_app  # noqa: F401
-from superset.extensions import (
+# pandas >= 2.2 advertises a minimum SQLAlchemy of 2.0 and silently ignores
+# older installations, breaking DataFrame.to_sql / read_sql with SQLAlchemy
+# 1.4 engines. Its SQL layer still works with 1.4, so restore support until
+# Superset itself requires SQLAlchemy >= 2. Must run before any pandas SQL IO.
+from superset.utils.pandas_sqlalchemy_compat import (  # noqa: E402
+    restore_pandas_sqlalchemy_support,
+)
+
+restore_pandas_sqlalchemy_support()
+
+from superset.app import create_app  # noqa: E402, F401
+from superset.extensions import (  # noqa: E402
     appbuilder,  # noqa: F401
     cache_manager,
     db,  # noqa: F401

superset/utils/pandas_sqlalchemy_compat.py

@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Compatibility shim letting pandas >= 2.2 use SQLAlchemy 1.4 engines.
+
+pandas 2.2 raised its advertised minimum SQLAlchemy version to 2.0 as a
+support-policy change. When an older SQLAlchemy is installed, pandas does not
+fail loudly: ``pandas.io.sql`` silently pretends SQLAlchemy is absent, treats
+Engine/Connection arguments as raw DBAPI connections, and falls back to its
+sqlite-only code path, breaking every ``DataFrame.to_sql`` / ``read_sql``
+call site (dataset uploads, example data loading, annotation queries, filter
+values).
+
+The pandas SQL layer itself still works with SQLAlchemy 1.4 because it only
+uses the API subset common to SQLAlchemy 1.4 and 2.x. Lowering the advertised
+minimum back to the pandas 2.1 value restores the working behavior.
+
+This module is obsolete once Superset requires SQLAlchemy >= 2; at that point
+the patch becomes a no-op and the module (and its call site in
+``superset/__init__.py``) can be deleted.
+"""
+
+import logging
+
+import sqlalchemy
+from packaging.version import Version
+
+logger = logging.getLogger(__name__)
+
+# The last pandas release line to support SQLAlchemy 1.4 (pandas 2.1)
+# required at least this version.
+_SQLALCHEMY_MINIMUM = "1.4.16"
+
+
+def restore_pandas_sqlalchemy_support() -> None:
+    """Lower pandas' advertised SQLAlchemy minimum so 1.4 engines work.
+
+    Only applies when the installed SQLAlchemy predates 2.0 and pandas
+    advertises a 2.x minimum; in every other combination this is a no-op.
+    Safe to call multiple times.
+    """
+    if Version(sqlalchemy.__version__) >= Version("2.0.0"):
+        # pandas supports SQLAlchemy 2.x natively; nothing to patch.
+        return
+
+    try:
+        from pandas.compat import _optional
+    except ImportError:
+        # The private module moved in a newer pandas; SQL IO with a pre-2.0
+        # SQLAlchemy will misbehave, so make the situation diagnosable.
+        logger.warning(
+            "Could not adjust pandas' minimum SQLAlchemy version; "
+            "DataFrame.to_sql/read_sql may not accept SQLAlchemy %s engines",
+            sqlalchemy.__version__,
+        )
+        return
+
+    advertised = _optional.VERSIONS.get("sqlalchemy")
+    if advertised and Version(advertised) > Version(_SQLALCHEMY_MINIMUM):
+        _optional.VERSIONS["sqlalchemy"] = _SQLALCHEMY_MINIMUM
+        logger.debug(
+            "Lowered pandas' minimum SQLAlchemy version from %s to %s so "
+            "pandas SQL IO keeps working with the installed SQLAlchemy %s",
+            advertised,
+            _SQLALCHEMY_MINIMUM,
+            sqlalchemy.__version__,
+        )

tests/unit_tests/utils/pandas_sqlalchemy_compat_test.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import pandas as pd
+from sqlalchemy import create_engine, types
+
+from superset.utils.pandas_sqlalchemy_compat import (
+    restore_pandas_sqlalchemy_support,
+)
+
+
+def test_to_sql_accepts_sqlalchemy_engine_and_dtypes() -> None:
+    """
+    ``DataFrame.to_sql`` must accept a SQLAlchemy engine plus SQLAlchemy
+    ``dtype`` objects regardless of the installed pandas/SQLAlchemy combo.
+
+    This is the exact call shape used by dataset uploads
+    (``BaseEngineSpec.df_to_sql``), example data loading, and the test data
+    loaders; it breaks when pandas silently rejects the installed SQLAlchemy
+    as too old (pandas >= 2.2 with SQLAlchemy 1.x) and no compat shim is
+    applied.
+    """
+    restore_pandas_sqlalchemy_support()
+
+    engine = create_engine("sqlite://")
+    df = pd.DataFrame(
+        {
+            "name": ["a", "b"],
+            "num": [1, 2],
+            "ds": pd.to_datetime(["2021-01-01", "2021-01-02"]),
+        }
+    )
+    df.to_sql(
+        "birth_names",
+        engine,
+        index=False,
+        dtype={"ds": types.DateTime(), "name": types.String(255)},
+        method="multi",
+        chunksize=100,
+    )
+    df.to_sql("birth_names", engine, index=False, if_exists="replace")
+
+    result = pd.read_sql_query("SELECT name, num FROM birth_names", engine)
+    assert result["name"].tolist() == ["a", "b"]
+    assert result["num"].tolist() == [1, 2]
+
+
+def test_restore_pandas_sqlalchemy_support_is_idempotent() -> None:
+    from pandas.compat import _optional
+
+    restore_pandas_sqlalchemy_support()
+    first = _optional.VERSIONS.get("sqlalchemy")
+    restore_pandas_sqlalchemy_support()
+    assert _optional.VERSIONS.get("sqlalchemy") == first