From 770dd5caeff5c26bad2f44c3adabfc67aaa3cf3a Mon Sep 17 00:00:00 2001 From: Superset Dev Date: Fri, 28 Aug 2026 18:28:09 -0700 Subject: [PATCH] feat(ci): expand testcontainers coverage to databend, risingwave, firebird, ydb, oceanbase, vertica Stacked on feat/testcontainers-nightly-only-gating. All six extras already existed in pyproject.toml. oceanbase and vertica run nightly_only: true (heavy first-boot and a ~12GB RAM floor, respectively), so they don't run per-PR; databend/risingwave/firebird/ydb run on every PR like the rest of this suite. - oceanbase_py pins sqlalchemy-utils<0.39, which conflicts outright with Superset's own sqlalchemy-utils==0.42.1 pin -- kept out of the baseline dev install (same reason as db2's ibm-db-sa) and installed on demand, --no-deps, only for its own CI leg (it never actually imports sqlalchemy_utils itself, so the version mismatch is inert at runtime). - databend: connects to the local standalone image's builtin `root` user (no password) with sslmode=disable, since Superset's default encryption_parameters assume TLS the local image doesn't have. - risingwave: RisingWave's storage engine checkpoints asynchronously -- a SELECT immediately after INSERT can see zero rows without an explicit FLUSH (confirmed on a real instance). Uses the shared _pagination.py helper's after_insert hook (originally added for CrateDB) to do that. - firebird: sqlalchemy-firebird's driver is a pure-Python ctypes wrapper (py3-none-any wheel, confirmed by downloading it directly) that dynamically loads the native libfbclient from the host rather than bundling it -- CI installs that system package on demand. Also confirms in the test docstring that FirebirdEngineSpec's `limit_method = LimitMethod.FETCH_MANY` (comment: "uses FIRST to limit") is stale against the modern driver, which compiles real ROWS-based pagination. - ydb: needed three real fixes to make a generic DockerContainer usable at all. (1) YDB's gRPC client does endpoint discovery and reconnects to whatever the server reports, which by default is the container's own internal Docker hostname -- fixed by binding the same port on the host as inside the container and advertising "localhost" as the container's own hostname, so the discovered endpoint is actually reachable. (2) The gRPC port opens before storage pools are fully initialized, so an early CREATE TABLE fails; the fixture retries a real metadata.create_all() probe rather than trusting the open port. (3) YDB rejects DDL inside an explicit transaction ("Scheme operations cannot be executed inside transaction") -- confirmed this only affects a raw text("CREATE TABLE..."), not metadata.create_all()'s own DDL execution path, which already does the right thing. --- .github/workflows/testcontainers.yml | 38 +++++ requirements/development.in | 21 ++- requirements/development.txt | 63 ++++++++ .../db_engine_specs/test_databend.py | 114 ++++++++++++++ .../db_engine_specs/test_firebird.py | 136 ++++++++++++++++ .../db_engine_specs/test_oceanbase.py | 137 ++++++++++++++++ .../db_engine_specs/test_risingwave.py | 127 +++++++++++++++ .../db_engine_specs/test_vertica.py | 121 ++++++++++++++ .../db_engine_specs/test_ydb.py | 148 ++++++++++++++++++ 9 files changed, 898 insertions(+), 7 deletions(-) create mode 100644 tests/testcontainers/db_engine_specs/test_databend.py create mode 100644 tests/testcontainers/db_engine_specs/test_firebird.py create mode 100644 tests/testcontainers/db_engine_specs/test_oceanbase.py create mode 100644 tests/testcontainers/db_engine_specs/test_risingwave.py create mode 100644 tests/testcontainers/db_engine_specs/test_vertica.py create mode 100644 tests/testcontainers/db_engine_specs/test_ydb.py diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index 297b696b9ad..2dff522e69d 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -90,6 +90,25 @@ jobs: # database -- wider margin until real CI data says otherwise. - dialect: starrocks timeout: 15 + - dialect: databend + timeout: 10 + - dialect: risingwave + timeout: 10 + - dialect: firebird + timeout: 10 + - dialect: ydb + timeout: 10 + # OceanBase bootstraps a distributed-style cluster even in + # single-node MODE=MINI, and Vertica Community Edition has a + # well-documented ~12GB RAM floor to even start -- both too heavy + # for every PR's CI budget, so both run on the nightly cron / + # manual dispatch only. + - dialect: oceanbase + timeout: 20 + nightly_only: true + - dialect: vertica + timeout: 20 + nightly_only: true timeout-minutes: ${{ matrix.timeout }} env: PYTHONPATH: ${{ github.workspace }} @@ -116,6 +135,25 @@ jobs: # instead, only for this leg of the matrix. if: matrix.dialect == 'db2' run: uv pip install --system -e .[db2] + - name: Install oceanbase driver (oceanbase_py) + # oceanbase_py pins sqlalchemy-utils>=0.38.3,<0.39, which conflicts + # outright with Superset's own sqlalchemy-utils==0.42.1 pin -- kept + # out of the baseline dev install for the same reason as db2 above. + # --no-deps sidesteps that pin entirely: this job only needs + # oceanbase_py's dialect module importable, not its sqlalchemy-utils + # dependency satisfied, since nothing here calls into it. + if: matrix.dialect == 'oceanbase' + run: uv pip install --system --no-deps -e .[oceanbase] + - name: Install Firebird client library (libfbclient2) + # sqlalchemy-firebird's driver (firebird-driver) is a pure-Python + # ctypes wrapper (its wheel is py3-none-any) that dynamically loads + # the native Firebird client library from the host at import time + # -- it doesn't bundle that library itself, so it has to come from + # the system package manager, only for this leg of the matrix. + if: matrix.dialect == 'firebird' + run: | + sudo apt-get update + sudo apt-get install -y libfbclient2 - name: Run testcontainers db_engine_specs tests (${{ matrix.dialect }}) # A job-level `if:` can't reference `matrix` (only github/inputs/ # needs/vars are available there), so the nightly_only skip has to diff --git a/requirements/development.in b/requirements/development.in index cc686c40cc4..2766bf35ab0 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -16,16 +16,20 @@ # specific language governing permissions and limitations # under the License. # --e .[development,bigquery,clickhouse,cockroachdb,crate,druid,duckdb,elasticsearch,fastmcp,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,starrocks,trino,thumbnails] +-e .[development,bigquery,clickhouse,cockroachdb,crate,databend,druid,duckdb,elasticsearch,fastmcp,firebird,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,risingwave,starrocks,trino,thumbnails,vertica,ydb] -e ./superset-extensions-cli[test] # testcontainers-backed db_engine_specs tests (tests/testcontainers/) -- # see .github/workflows/testcontainers.yml # -# `db2` (the `ibm-db-sa`/`ibm-db` driver) is deliberately left out of the -# baseline dev install above: `ibm-db` ships no Linux arm64 wheel, so -# including it here breaks the multi-platform (amd64+arm64) dev Docker -# image build. The testcontainers CI job installs it on demand, only for -# the db2 matrix leg -- see .github/workflows/testcontainers.yml. +# `db2` (the `ibm-db-sa`/`ibm-db` driver) and `oceanbase` (the `oceanbase_py` +# driver) are both deliberately left out of the baseline dev install above: +# `ibm-db` ships no Linux arm64 wheel, breaking the multi-platform +# (amd64+arm64) dev Docker image build; `oceanbase_py` pins +# `sqlalchemy-utils>=0.38.3,<0.39`, which conflicts outright with Superset's +# own `sqlalchemy-utils==0.42.1` pin -- there's no version of both that can +# coexist in one resolved environment. Both testcontainers CI jobs install +# their driver on demand instead, only for their own matrix leg -- see +# .github/workflows/testcontainers.yml. # # mariadb/timescaledb/yugabytedb need no testcontainers extra of their own: # they reuse the postgres/mysql container classes pointed at a different @@ -36,5 +40,8 @@ # pulled in above via the clickhouse extra, is all the test needs), and # StarRocks has no dedicated testcontainers module at all -- its test uses # a generic DockerContainer plus the same mysqlclient the mysql extra -# already provides. +# already provides. databend/risingwave/firebird/ydb/vertica are the same +# story: none has a dedicated testcontainers module, so each test uses a +# generic DockerContainer plus whatever driver its own extra above already +# provides. testcontainers[cockroachdb,cratedb,mongodb,mssql,mysql,oracle,postgres,trino]>=4.15.0,<5 diff --git a/requirements/development.txt b/requirements/development.txt index 559663aeb33..8cf74e28ebe 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -12,6 +12,12 @@ # apache-superset aiofile==3.9.0 # via py-key-value-aio +aiohappyeyeballs==2.7.1 + # via aiohttp +aiohttp==3.14.3 + # via ydb +aiosignal==1.4.0 + # via aiohttp alembic==1.15.2 # via # -c requirements/base-constraint.txt @@ -50,6 +56,7 @@ asyncmy2==0.2.21 attrs==25.3.0 # via # -c requirements/base-constraint.txt + # aiohttp # cattrs # cyclopts # jsonschema @@ -209,6 +216,10 @@ cycler==0.12.1 # via matplotlib cyclopts==4.2.4 # via fastmcp-slim +databend-driver==0.34.2 + # via databend-sqlalchemy +databend-sqlalchemy==0.5.5 + # via apache-superset db-dtypes==1.3.1 # via pandas-gbq defusedxml==0.7.1 @@ -271,6 +282,10 @@ filelock==3.20.3 # via # -c requirements/base-constraint.txt # virtualenv +firebird-base==2.0.3 + # via firebird-driver +firebird-driver==2.0.3 + # via sqlalchemy-firebird flask==2.3.3 # via # -c requirements/base-constraint.txt @@ -351,6 +366,10 @@ fonttools==4.60.2 # via matplotlib freezegun==1.5.1 # via apache-superset +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal future==1.0.0 # via pyhive geographiclib==2.0 @@ -415,6 +434,7 @@ grpcio==1.83.0 # apache-superset # google-api-core # grpcio-status + # ydb grpcio-status==1.60.1 # via google-api-core gunicorn==26.2.0 @@ -457,6 +477,7 @@ idna==3.15 # httpx # requests # url-normalize + # yarl importlib-metadata==8.7.0 # via # keyring @@ -599,6 +620,10 @@ msgspec==0.19.0 # via # -c requirements/base-constraint.txt # flask-session +multidict==6.7.1 + # via + # aiohttp + # yarl mysqlclient==2.2.8 # via apache-superset nh3==0.3.6 @@ -671,6 +696,8 @@ packaging==25.0 # pytest # shillelagh # sqlalchemy-bigquery + # sqlalchemy-firebird + # ydb pandas==2.3.3 # via # -c requirements/base-constraint.txt @@ -732,16 +759,22 @@ prompt-toolkit==3.0.51 # via # -c requirements/base-constraint.txt # click-repl +propcache==0.5.2 + # via + # aiohttp + # yarl prophet==1.4.0 # via apache-superset proto-plus==1.25.0 # via google-api-core protobuf==5.29.6 # via + # firebird-base # google-api-core # googleapis-common-protos # grpcio-status # proto-plus + # ydb psutil==6.1.0 # via # apache-superset @@ -877,6 +910,7 @@ python-dateutil==2.9.0.post0 # botocore # celery # croniter + # firebird-driver # flask-appbuilder # freezegun # google-cloud-bigquery @@ -887,6 +921,7 @@ python-dateutil==2.9.0.post0 # pyhive # shillelagh # trino + # vertica-python python-dotenv==1.2.2 # via # -c requirements/base-constraint.txt @@ -1003,6 +1038,7 @@ six==1.17.0 # prison # python-dateutil # rfc3339-validator + # vertica-python # wtforms-json slack-sdk==3.43.0 # via @@ -1016,6 +1052,7 @@ sqlalchemy==2.0.52 # alembic # apache-superset # apache-superset-core + # databend-sqlalchemy # duckdb-engine # elasticsearch-dbapi # flask-appbuilder @@ -1026,10 +1063,13 @@ sqlalchemy==2.0.52 # sqlalchemy-cockroachdb # sqlalchemy-continuum # sqlalchemy-cratedb + # sqlalchemy-firebird # sqlalchemy-monetdb + # sqlalchemy-risingwave # sqlalchemy-utils # starrocks # testcontainers + # ydb-sqlalchemy sqlalchemy-bigquery==1.17.2 # via apache-superset sqlalchemy-cockroachdb==2.0.4 @@ -1042,19 +1082,26 @@ sqlalchemy-cratedb==0.43.1 # via # apache-superset # testcontainers +sqlalchemy-firebird==2.2.0 + # via apache-superset sqlalchemy-monetdb==2.1.0 # via apache-superset +sqlalchemy-risingwave==2.1.0 + # via apache-superset sqlalchemy-utils==0.42.1 # via # -c requirements/base-constraint.txt # apache-superset # apache-superset-core # flask-appbuilder +sqlalchemy-vertica-python==0.6.3 + # via apache-superset sqlglot==30.17.0 # via # -c requirements/base-constraint.txt # apache-superset # apache-superset-core + # ydb-sqlglot-plugin sqloxide==0.1.51 # via apache-superset sse-starlette==3.0.2 @@ -1096,6 +1143,8 @@ trino==0.339.0 typing-extensions==4.16.0 # via # -c requirements/base-constraint.txt + # aiohttp + # aiosignal # alembic # anyio # apache-superset @@ -1160,6 +1209,8 @@ verlib2==0.3.2 # via # crate # sqlalchemy-cratedb +vertica-python==1.4.0 + # via sqlalchemy-vertica-python vine==5.1.0 # via # -c requirements/base-constraint.txt @@ -1214,6 +1265,18 @@ xlsxwriter==3.2.9 # -c requirements/base-constraint.txt # apache-superset # pandas +yarl==1.24.5 + # via aiohttp +ydb==3.31.4 + # via + # ydb-dbapi + # ydb-sqlalchemy +ydb-dbapi==0.1.23 + # via ydb-sqlalchemy +ydb-sqlalchemy==0.1.22 + # via apache-superset +ydb-sqlglot-plugin==0.2.8 + # via apache-superset zipp==3.23.0 # via importlib-metadata zope-event==5.0 diff --git a/tests/testcontainers/db_engine_specs/test_databend.py b/tests/testcontainers/db_engine_specs/test_databend.py new file mode 100644 index 00000000000..e4f4f1f7662 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_databend.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.databend against a real Databend instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +Databend has no dedicated testcontainers module, so this uses a generic +DockerContainer against the official `datafuselabs/databend` standalone +image. Superset's DatabendEngineSpec defaults to `sslmode=require` +(`encryption_parameters`), but the local standalone image has no TLS +listener, so this connects with `sslmode=disable` explicitly. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.databend import DatabendEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("databend_sqlalchemy") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +HTTP_PORT = 8000 +DBNAME = "default" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("datafuselabs/databend") + container.with_exposed_ports(HTTP_PORT) + # The image's own startup banner documents this exact line as proof its + # HTTP query endpoint is bound and ready. + container.waiting_for(LogMessageWaitStrategy(f"listened at 0.0.0.0:{HTTP_PORT}")) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(HTTP_PORT) + # "root" with no password is the image's builtin user -- confirmed + # directly against a running container, not from the image's own + # doc text, which only shows ${USER}/${PASSWORD} placeholders. + yield create_engine(f"databend://root:@{host}:{port}/{DBNAME}?sslmode=disable") + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + DatabendEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = DatabendEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = DatabendEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_firebird.py b/tests/testcontainers/db_engine_specs/test_firebird.py new file mode 100644 index 00000000000..9e439b7b52a --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_firebird.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.firebird against a real Firebird instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +Firebird references a database *file* rather than a server-managed named +database -- the connection URI is `firebird://user:pass@host:port/`, +where is the path to a .fdb file on the server. This uses the +well-known `jacobalberty/firebird` image, which creates that file (per +FIREBIRD_DATABASE) under /firebird/data on first boot. + +FirebirdEngineSpec sets `limit_method = LimitMethod.FETCH_MANY` with a +comment claiming Firebird "uses FIRST to limit" -- stale relative to the +modern sqlalchemy-firebird driver, which compiles real ROWS-based +pagination (confirmed via an offline dialect compile: `SELECT ... ROWS +4 + 1 TO 4 + 3`, correctly ordered, not a Trino-style bug). That staleness +affects what Superset's own query layer emits, not what this suite's +direct dialect-compilation check exercises. + +Could not be verified against a real running instance in this +environment: `firebird-driver` is a pure-Python ctypes wrapper (its wheel +is `py3-none-any`, confirmed by downloading it directly) that dynamically +loads the native Firebird client library (`libfbclient`) from the host at +import time -- it doesn't bundle that library itself. This machine has no +Homebrew formula or straightforward install path for it. The container +itself was confirmed to start and pass its own healthcheck; CI installs +the `libfbclient2` system package separately (see +.github/workflows/testcontainers.yml) for the actual client-library +dependency this driver needs. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.firebird import FirebirdEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("firebird.driver") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import HealthcheckWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 3050 +PASSWORD = "masterkey" # noqa: S105 -- fixed test-fixture password, not a secret +DB_FILE = "test.fdb" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("jacobalberty/firebird") + container.with_exposed_ports(PORT) + container.with_env("ISC_PASSWORD", PASSWORD) + container.with_env("FIREBIRD_DATABASE", DB_FILE) + # The image logs nothing beyond a single startup banner line and never + # prints a distinct "ready" message -- it ships its own Docker + # HEALTHCHECK instead, confirmed via `docker ps` reporting (healthy). + container.waiting_for(HealthcheckWaitStrategy()) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine( + f"firebird://sysdba:{PASSWORD}@{host}:{port}//firebird/data/{DB_FILE}" + ) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + FirebirdEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = FirebirdEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = FirebirdEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_oceanbase.py b/tests/testcontainers/db_engine_specs/test_oceanbase.py new file mode 100644 index 00000000000..f1b395046d1 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_oceanbase.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.oceanbase against a real OceanBase instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml, +on the nightly cron / manual dispatch only (see `nightly_only: true` on +this dialect's matrix entry) -- OceanBase bootstraps a distributed-style +cluster even in single-node MODE=MINI, a substantially heavier first-boot +than a single-process database, not a good fit for every PR's CI budget. + +OceanBaseEngineSpec extends MySQLEngineSpec and its dialect +(oceanbase_py.sqlalchemy.dialect.OceanBaseDialect) extends +MySQLDialect_mysqldb directly with no custom DDL or LIMIT/OFFSET compiler, +so this follows the same mysqlclient-based pattern as MariaDB/MySQL/ +StarRocks in this suite -- including the same "localhost" -> "127.0.0.1" +fix MySQLdb needs on native Linux Docker. + +Could not be verified locally in this environment: mysqlclient (MySQLdb) +has a pre-existing, unrelated native-library linking issue against this +machine's Homebrew-installed libmysqlclient, and this dialect wasn't +pulled/run locally at all given its heavier resource footprint -- CI-only +verification, matching the nightly_only gating. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine, URL + +from superset.db_engine_specs.oceanbase import OceanBaseEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("oceanbase_py") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 2881 +PASSWORD = "pilot" # noqa: S105 -- fixed test-fixture password, not a secret + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("oceanbase/oceanbase-ce") + container.with_exposed_ports(PORT) + container.with_env("MODE", "MINI") + container.with_env("OB_TENANT_PASSWORD", PASSWORD) + container.waiting_for(LogMessageWaitStrategy("boot success!")) + + with container: + host = container.get_container_host_ip() + if host == "localhost": + host = "127.0.0.1" + port = container.get_exposed_port(PORT) + # OceanBase usernames for a MySQL-mode tenant use "user@tenant" + # (e.g. "root@test"), a literal "@" that URL.create() percent-encodes + # correctly -- an f-string would produce a second "@" that breaks + # the URL's own host/user boundary parsing. + yield create_engine( + URL.create( + "oceanbase", + username="root@test", + password=PASSWORD, + host=host, + port=int(port), + database="test", + ) + ) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + OceanBaseEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = OceanBaseEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = OceanBaseEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_risingwave.py b/tests/testcontainers/db_engine_specs/test_risingwave.py new file mode 100644 index 00000000000..db269f0f36c --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_risingwave.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.risingwave against a real RisingWave instance, spun +up on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +RisingWave speaks the Postgres wire protocol, but doesn't run the real +Postgres server binary or its POSTGRES_PASSWORD-style bootstrap env vars, +so this can't reuse `PostgresContainer` the way TimescaleDB/YugabyteDB do +-- it needs a generic DockerContainer against the official +`risingwavelabs/risingwave` single-binary playground image instead. +`RisingWaveDbEngineSpec` extends `PostgresEngineSpec`, and +`sqlalchemy-risingwave`'s dialect is a genuine subclass of SQLAlchemy's own +Postgres dialect (via psycopg2), so DDL/pagination compile with standard +Postgres semantics -- no ClickHouse-style mandatory table option needed. +""" + +import re +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, + text, +) +from sqlalchemy.engine import Connection, Engine + +from superset.db_engine_specs.risingwave import RisingWaveDbEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("sqlalchemy_risingwave") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 4566 + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("risingwavelabs/risingwave") + container.with_exposed_ports(PORT) + container.with_command("playground") + # The actual startup banner reads "RisingWave standalone mode is + # ready." -- confirmed against a real container's logs. + container.waiting_for( + LogMessageWaitStrategy(re.compile("RisingWave standalone mode is ready")) + ) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine(f"risingwave://root@{host}:{port}/dev") + + +def _flush(conn: Connection) -> None: + # RisingWave's storage engine checkpoints asynchronously: without an + # explicit FLUSH, a SELECT immediately after INSERT can see zero rows + # -- confirmed against a real instance (a bare INSERT commits fine, but + # the data isn't visible to a subsequent query until flushed). + conn.execute(text("FLUSH")) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine, after_insert=_flush) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + RisingWaveDbEngineSpec.get_columns wraps a real SQLAlchemy Inspector; + this exercises that against actual server-reported column metadata + rather than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = RisingWaveDbEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = RisingWaveDbEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_vertica.py b/tests/testcontainers/db_engine_specs/test_vertica.py new file mode 100644 index 00000000000..dc446971a2e --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_vertica.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.vertica against a real Vertica instance, spun up on +demand via testcontainers. Run via .github/workflows/testcontainers.yml, +on the nightly cron / manual dispatch only (see `nightly_only: true` on +this dialect's matrix entry) -- Vertica Community Edition has a +well-documented ~12GB RAM floor to even start (a license-tier check baked +into the image), well above what a per-PR CI runner should be expected to +provide. + +VerticaEngineSpec extends PostgresBaseEngineSpec, and +sqla_vertica_python.vertica_python.VerticaDialect is a direct subclass of +SQLAlchemy's own postgresql.PGDialect with only an index-syntax override +-- no custom DDL or LIMIT/OFFSET compiler, so a bare +Column(..., primary_key=True) table and standard LIMIT/OFFSET pagination +both compile with plain Postgres semantics (Vertica auto-creates a +default superprojection; no explicit projection/segmentation clause is +needed the way ClickHouse needs an ENGINE=). + +Not verified locally in this environment: Vertica CE's RAM/image weight +was judged not worth pulling on a resource-constrained local machine per +session guidance -- CI-only verification, matching the nightly_only +gating. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.vertica import VerticaEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("vertica_python") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 5433 + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("vertica/vertica-ce") + container.with_exposed_ports(PORT) + container.waiting_for(LogMessageWaitStrategy("Vertica is now running")) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine(f"vertica+vertica_python://dbadmin@{host}:{port}/VMart") + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + VerticaEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = VerticaEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = VerticaEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_ydb.py b/tests/testcontainers/db_engine_specs/test_ydb.py new file mode 100644 index 00000000000..8e0d72a7a54 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_ydb.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.ydb against a real YDB instance, spun up on demand +via testcontainers. Run via .github/workflows/testcontainers.yml. + +YDB has no dedicated testcontainers module, so this uses a generic +DockerContainer against the official `ydbplatform/local-ydb` image, which +needs no auth for local/anonymous access -- YDBEngineSpec's own +`sqlalchemy_uri_placeholder` ("ydb://{host}:{port}/{database_name}") has +no username/password at all. +""" + +import time +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.ydb import YDBEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("ydb_sqlalchemy") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import PortWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +GRPC_PORT = 2136 +DATABASE = "/local" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("ydbplatform/local-ydb") + container.with_exposed_ports(GRPC_PORT) + # YDB's gRPC client does endpoint discovery: it asks the server for its + # "real" endpoints and reconnects to whatever comes back, rather than + # just using the address it was originally given. By default that's + # the container's own internal Docker hostname (e.g. "6abbb4bb0ab7"), + # which isn't reachable from the host. Binding the same port number on + # the host as inside the container, plus advertising "localhost" as + # the container's own hostname, makes the discovered endpoint + # ("localhost:2136") actually resolve to something reachable. + container.with_bind_ports(GRPC_PORT, GRPC_PORT) + container.with_kwargs(hostname="localhost") + container.with_env("YDB_USE_IN_MEMORY_PDISKS", "true") + container.waiting_for(PortWaitStrategy(GRPC_PORT)) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(GRPC_PORT) + eng = create_engine(f"yql://{host}:{port}{DATABASE}") + + # The gRPC port opens, and even a bare SELECT succeeds, before YDB's + # storage pools are fully initialized -- an actual CREATE TABLE can + # still fail with "database doesn't have storage pools at all to + # create tablet channels" (confirmed on a real instance). Probe with + # metadata.create_all()/drop_all() specifically, the same call the + # real tests below make: a raw `text("CREATE TABLE ...")` hits a + # separate, unrelated error ("Scheme operations cannot be executed + # inside transaction") that create_all()'s own DDL execution path + # doesn't, even with AUTOCOMMIT set on a manually-opened connection. + probe_metadata = MetaData() + SATable("pilot_ready", probe_metadata, Column("id", Integer, primary_key=True)) + last_error: Exception | None = None + for _ in range(30): + try: + probe_metadata.create_all(eng) + probe_metadata.drop_all(eng) + break + except Exception as ex: # noqa: BLE001 -- retry on any not-ready-yet error + last_error = ex + time.sleep(2) + else: + raise RuntimeError( + "YDB never became ready to create and use a table" + ) from last_error + + yield eng + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + YDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = YDBEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = YDBEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer)