diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index 297b696b9ad..2dff522e69d 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -90,6 +90,25 @@ jobs: # database -- wider margin until real CI data says otherwise. - dialect: starrocks timeout: 15 + - dialect: databend + timeout: 10 + - dialect: risingwave + timeout: 10 + - dialect: firebird + timeout: 10 + - dialect: ydb + timeout: 10 + # OceanBase bootstraps a distributed-style cluster even in + # single-node MODE=MINI, and Vertica Community Edition has a + # well-documented ~12GB RAM floor to even start -- both too heavy + # for every PR's CI budget, so both run on the nightly cron / + # manual dispatch only. + - dialect: oceanbase + timeout: 20 + nightly_only: true + - dialect: vertica + timeout: 20 + nightly_only: true timeout-minutes: ${{ matrix.timeout }} env: PYTHONPATH: ${{ github.workspace }} @@ -116,6 +135,25 @@ jobs: # instead, only for this leg of the matrix. if: matrix.dialect == 'db2' run: uv pip install --system -e .[db2] + - name: Install oceanbase driver (oceanbase_py) + # oceanbase_py pins sqlalchemy-utils>=0.38.3,<0.39, which conflicts + # outright with Superset's own sqlalchemy-utils==0.42.1 pin -- kept + # out of the baseline dev install for the same reason as db2 above. + # --no-deps sidesteps that pin entirely: this job only needs + # oceanbase_py's dialect module importable, not its sqlalchemy-utils + # dependency satisfied, since nothing here calls into it. + if: matrix.dialect == 'oceanbase' + run: uv pip install --system --no-deps -e .[oceanbase] + - name: Install Firebird client library (libfbclient2) + # sqlalchemy-firebird's driver (firebird-driver) is a pure-Python + # ctypes wrapper (its wheel is py3-none-any) that dynamically loads + # the native Firebird client library from the host at import time + # -- it doesn't bundle that library itself, so it has to come from + # the system package manager, only for this leg of the matrix. + if: matrix.dialect == 'firebird' + run: | + sudo apt-get update + sudo apt-get install -y libfbclient2 - name: Run testcontainers db_engine_specs tests (${{ matrix.dialect }}) # A job-level `if:` can't reference `matrix` (only github/inputs/ # needs/vars are available there), so the nightly_only skip has to diff --git a/requirements/development.in b/requirements/development.in index cc686c40cc4..2766bf35ab0 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -16,16 +16,20 @@ # specific language governing permissions and limitations # under the License. # --e .[development,bigquery,clickhouse,cockroachdb,crate,druid,duckdb,elasticsearch,fastmcp,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,starrocks,trino,thumbnails] +-e .[development,bigquery,clickhouse,cockroachdb,crate,databend,druid,duckdb,elasticsearch,fastmcp,firebird,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,risingwave,starrocks,trino,thumbnails,vertica,ydb] -e ./superset-extensions-cli[test] # testcontainers-backed db_engine_specs tests (tests/testcontainers/) -- # see .github/workflows/testcontainers.yml # -# `db2` (the `ibm-db-sa`/`ibm-db` driver) is deliberately left out of the -# baseline dev install above: `ibm-db` ships no Linux arm64 wheel, so -# including it here breaks the multi-platform (amd64+arm64) dev Docker -# image build. The testcontainers CI job installs it on demand, only for -# the db2 matrix leg -- see .github/workflows/testcontainers.yml. +# `db2` (the `ibm-db-sa`/`ibm-db` driver) and `oceanbase` (the `oceanbase_py` +# driver) are both deliberately left out of the baseline dev install above: +# `ibm-db` ships no Linux arm64 wheel, breaking the multi-platform +# (amd64+arm64) dev Docker image build; `oceanbase_py` pins +# `sqlalchemy-utils>=0.38.3,<0.39`, which conflicts outright with Superset's +# own `sqlalchemy-utils==0.42.1` pin -- there's no version of both that can +# coexist in one resolved environment. Both testcontainers CI jobs install +# their driver on demand instead, only for their own matrix leg -- see +# .github/workflows/testcontainers.yml. # # mariadb/timescaledb/yugabytedb need no testcontainers extra of their own: # they reuse the postgres/mysql container classes pointed at a different @@ -36,5 +40,8 @@ # pulled in above via the clickhouse extra, is all the test needs), and # StarRocks has no dedicated testcontainers module at all -- its test uses # a generic DockerContainer plus the same mysqlclient the mysql extra -# already provides. +# already provides. databend/risingwave/firebird/ydb/vertica are the same +# story: none has a dedicated testcontainers module, so each test uses a +# generic DockerContainer plus whatever driver its own extra above already +# provides. testcontainers[cockroachdb,cratedb,mongodb,mssql,mysql,oracle,postgres,trino]>=4.15.0,<5 diff --git a/requirements/development.txt b/requirements/development.txt index 559663aeb33..8cf74e28ebe 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -12,6 +12,12 @@ # apache-superset aiofile==3.9.0 # via py-key-value-aio +aiohappyeyeballs==2.7.1 + # via aiohttp +aiohttp==3.14.3 + # via ydb +aiosignal==1.4.0 + # via aiohttp alembic==1.15.2 # via # -c requirements/base-constraint.txt @@ -50,6 +56,7 @@ asyncmy2==0.2.21 attrs==25.3.0 # via # -c requirements/base-constraint.txt + # aiohttp # cattrs # cyclopts # jsonschema @@ -209,6 +216,10 @@ cycler==0.12.1 # via matplotlib cyclopts==4.2.4 # via fastmcp-slim +databend-driver==0.34.2 + # via databend-sqlalchemy +databend-sqlalchemy==0.5.5 + # via apache-superset db-dtypes==1.3.1 # via pandas-gbq defusedxml==0.7.1 @@ -271,6 +282,10 @@ filelock==3.20.3 # via # -c requirements/base-constraint.txt # virtualenv +firebird-base==2.0.3 + # via firebird-driver +firebird-driver==2.0.3 + # via sqlalchemy-firebird flask==2.3.3 # via # -c requirements/base-constraint.txt @@ -351,6 +366,10 @@ fonttools==4.60.2 # via matplotlib freezegun==1.5.1 # via apache-superset +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal future==1.0.0 # via pyhive geographiclib==2.0 @@ -415,6 +434,7 @@ grpcio==1.83.0 # apache-superset # google-api-core # grpcio-status + # ydb grpcio-status==1.60.1 # via google-api-core gunicorn==26.2.0 @@ -457,6 +477,7 @@ idna==3.15 # httpx # requests # url-normalize + # yarl importlib-metadata==8.7.0 # via # keyring @@ -599,6 +620,10 @@ msgspec==0.19.0 # via # -c requirements/base-constraint.txt # flask-session +multidict==6.7.1 + # via + # aiohttp + # yarl mysqlclient==2.2.8 # via apache-superset nh3==0.3.6 @@ -671,6 +696,8 @@ packaging==25.0 # pytest # shillelagh # sqlalchemy-bigquery + # sqlalchemy-firebird + # ydb pandas==2.3.3 # via # -c requirements/base-constraint.txt @@ -732,16 +759,22 @@ prompt-toolkit==3.0.51 # via # -c requirements/base-constraint.txt # click-repl +propcache==0.5.2 + # via + # aiohttp + # yarl prophet==1.4.0 # via apache-superset proto-plus==1.25.0 # via google-api-core protobuf==5.29.6 # via + # firebird-base # google-api-core # googleapis-common-protos # grpcio-status # proto-plus + # ydb psutil==6.1.0 # via # apache-superset @@ -877,6 +910,7 @@ python-dateutil==2.9.0.post0 # botocore # celery # croniter + # firebird-driver # flask-appbuilder # freezegun # google-cloud-bigquery @@ -887,6 +921,7 @@ python-dateutil==2.9.0.post0 # pyhive # shillelagh # trino + # vertica-python python-dotenv==1.2.2 # via # -c requirements/base-constraint.txt @@ -1003,6 +1038,7 @@ six==1.17.0 # prison # python-dateutil # rfc3339-validator + # vertica-python # wtforms-json slack-sdk==3.43.0 # via @@ -1016,6 +1052,7 @@ sqlalchemy==2.0.52 # alembic # apache-superset # apache-superset-core + # databend-sqlalchemy # duckdb-engine # elasticsearch-dbapi # flask-appbuilder @@ -1026,10 +1063,13 @@ sqlalchemy==2.0.52 # sqlalchemy-cockroachdb # sqlalchemy-continuum # sqlalchemy-cratedb + # sqlalchemy-firebird # sqlalchemy-monetdb + # sqlalchemy-risingwave # sqlalchemy-utils # starrocks # testcontainers + # ydb-sqlalchemy sqlalchemy-bigquery==1.17.2 # via apache-superset sqlalchemy-cockroachdb==2.0.4 @@ -1042,19 +1082,26 @@ sqlalchemy-cratedb==0.43.1 # via # apache-superset # testcontainers +sqlalchemy-firebird==2.2.0 + # via apache-superset sqlalchemy-monetdb==2.1.0 # via apache-superset +sqlalchemy-risingwave==2.1.0 + # via apache-superset sqlalchemy-utils==0.42.1 # via # -c requirements/base-constraint.txt # apache-superset # apache-superset-core # flask-appbuilder +sqlalchemy-vertica-python==0.6.3 + # via apache-superset sqlglot==30.17.0 # via # -c requirements/base-constraint.txt # apache-superset # apache-superset-core + # ydb-sqlglot-plugin sqloxide==0.1.51 # via apache-superset sse-starlette==3.0.2 @@ -1096,6 +1143,8 @@ trino==0.339.0 typing-extensions==4.16.0 # via # -c requirements/base-constraint.txt + # aiohttp + # aiosignal # alembic # anyio # apache-superset @@ -1160,6 +1209,8 @@ verlib2==0.3.2 # via # crate # sqlalchemy-cratedb +vertica-python==1.4.0 + # via sqlalchemy-vertica-python vine==5.1.0 # via # -c requirements/base-constraint.txt @@ -1214,6 +1265,18 @@ xlsxwriter==3.2.9 # -c requirements/base-constraint.txt # apache-superset # pandas +yarl==1.24.5 + # via aiohttp +ydb==3.31.4 + # via + # ydb-dbapi + # ydb-sqlalchemy +ydb-dbapi==0.1.23 + # via ydb-sqlalchemy +ydb-sqlalchemy==0.1.22 + # via apache-superset +ydb-sqlglot-plugin==0.2.8 + # via apache-superset zipp==3.23.0 # via importlib-metadata zope-event==5.0 diff --git a/tests/testcontainers/db_engine_specs/test_databend.py b/tests/testcontainers/db_engine_specs/test_databend.py new file mode 100644 index 00000000000..e4f4f1f7662 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_databend.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.databend against a real Databend instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +Databend has no dedicated testcontainers module, so this uses a generic +DockerContainer against the official `datafuselabs/databend` standalone +image. Superset's DatabendEngineSpec defaults to `sslmode=require` +(`encryption_parameters`), but the local standalone image has no TLS +listener, so this connects with `sslmode=disable` explicitly. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.databend import DatabendEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("databend_sqlalchemy") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +HTTP_PORT = 8000 +DBNAME = "default" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("datafuselabs/databend") + container.with_exposed_ports(HTTP_PORT) + # The image's own startup banner documents this exact line as proof its + # HTTP query endpoint is bound and ready. + container.waiting_for(LogMessageWaitStrategy(f"listened at 0.0.0.0:{HTTP_PORT}")) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(HTTP_PORT) + # "root" with no password is the image's builtin user -- confirmed + # directly against a running container, not from the image's own + # doc text, which only shows ${USER}/${PASSWORD} placeholders. + yield create_engine(f"databend://root:@{host}:{port}/{DBNAME}?sslmode=disable") + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + DatabendEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = DatabendEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = DatabendEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_firebird.py b/tests/testcontainers/db_engine_specs/test_firebird.py new file mode 100644 index 00000000000..9e439b7b52a --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_firebird.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.firebird against a real Firebird instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +Firebird references a database *file* rather than a server-managed named +database -- the connection URI is `firebird://user:pass@host:port/`, +where is the path to a .fdb file on the server. This uses the +well-known `jacobalberty/firebird` image, which creates that file (per +FIREBIRD_DATABASE) under /firebird/data on first boot. + +FirebirdEngineSpec sets `limit_method = LimitMethod.FETCH_MANY` with a +comment claiming Firebird "uses FIRST to limit" -- stale relative to the +modern sqlalchemy-firebird driver, which compiles real ROWS-based +pagination (confirmed via an offline dialect compile: `SELECT ... ROWS +4 + 1 TO 4 + 3`, correctly ordered, not a Trino-style bug). That staleness +affects what Superset's own query layer emits, not what this suite's +direct dialect-compilation check exercises. + +Could not be verified against a real running instance in this +environment: `firebird-driver` is a pure-Python ctypes wrapper (its wheel +is `py3-none-any`, confirmed by downloading it directly) that dynamically +loads the native Firebird client library (`libfbclient`) from the host at +import time -- it doesn't bundle that library itself. This machine has no +Homebrew formula or straightforward install path for it. The container +itself was confirmed to start and pass its own healthcheck; CI installs +the `libfbclient2` system package separately (see +.github/workflows/testcontainers.yml) for the actual client-library +dependency this driver needs. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.firebird import FirebirdEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("firebird.driver") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import HealthcheckWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 3050 +PASSWORD = "masterkey" # noqa: S105 -- fixed test-fixture password, not a secret +DB_FILE = "test.fdb" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("jacobalberty/firebird") + container.with_exposed_ports(PORT) + container.with_env("ISC_PASSWORD", PASSWORD) + container.with_env("FIREBIRD_DATABASE", DB_FILE) + # The image logs nothing beyond a single startup banner line and never + # prints a distinct "ready" message -- it ships its own Docker + # HEALTHCHECK instead, confirmed via `docker ps` reporting (healthy). + container.waiting_for(HealthcheckWaitStrategy()) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine( + f"firebird://sysdba:{PASSWORD}@{host}:{port}//firebird/data/{DB_FILE}" + ) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + FirebirdEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = FirebirdEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = FirebirdEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_oceanbase.py b/tests/testcontainers/db_engine_specs/test_oceanbase.py new file mode 100644 index 00000000000..f1b395046d1 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_oceanbase.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.oceanbase against a real OceanBase instance, spun up +on demand via testcontainers. Run via .github/workflows/testcontainers.yml, +on the nightly cron / manual dispatch only (see `nightly_only: true` on +this dialect's matrix entry) -- OceanBase bootstraps a distributed-style +cluster even in single-node MODE=MINI, a substantially heavier first-boot +than a single-process database, not a good fit for every PR's CI budget. + +OceanBaseEngineSpec extends MySQLEngineSpec and its dialect +(oceanbase_py.sqlalchemy.dialect.OceanBaseDialect) extends +MySQLDialect_mysqldb directly with no custom DDL or LIMIT/OFFSET compiler, +so this follows the same mysqlclient-based pattern as MariaDB/MySQL/ +StarRocks in this suite -- including the same "localhost" -> "127.0.0.1" +fix MySQLdb needs on native Linux Docker. + +Could not be verified locally in this environment: mysqlclient (MySQLdb) +has a pre-existing, unrelated native-library linking issue against this +machine's Homebrew-installed libmysqlclient, and this dialect wasn't +pulled/run locally at all given its heavier resource footprint -- CI-only +verification, matching the nightly_only gating. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine, URL + +from superset.db_engine_specs.oceanbase import OceanBaseEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("oceanbase_py") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 2881 +PASSWORD = "pilot" # noqa: S105 -- fixed test-fixture password, not a secret + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("oceanbase/oceanbase-ce") + container.with_exposed_ports(PORT) + container.with_env("MODE", "MINI") + container.with_env("OB_TENANT_PASSWORD", PASSWORD) + container.waiting_for(LogMessageWaitStrategy("boot success!")) + + with container: + host = container.get_container_host_ip() + if host == "localhost": + host = "127.0.0.1" + port = container.get_exposed_port(PORT) + # OceanBase usernames for a MySQL-mode tenant use "user@tenant" + # (e.g. "root@test"), a literal "@" that URL.create() percent-encodes + # correctly -- an f-string would produce a second "@" that breaks + # the URL's own host/user boundary parsing. + yield create_engine( + URL.create( + "oceanbase", + username="root@test", + password=PASSWORD, + host=host, + port=int(port), + database="test", + ) + ) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + OceanBaseEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = OceanBaseEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = OceanBaseEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_risingwave.py b/tests/testcontainers/db_engine_specs/test_risingwave.py new file mode 100644 index 00000000000..db269f0f36c --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_risingwave.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.risingwave against a real RisingWave instance, spun +up on demand via testcontainers. Run via .github/workflows/testcontainers.yml. + +RisingWave speaks the Postgres wire protocol, but doesn't run the real +Postgres server binary or its POSTGRES_PASSWORD-style bootstrap env vars, +so this can't reuse `PostgresContainer` the way TimescaleDB/YugabyteDB do +-- it needs a generic DockerContainer against the official +`risingwavelabs/risingwave` single-binary playground image instead. +`RisingWaveDbEngineSpec` extends `PostgresEngineSpec`, and +`sqlalchemy-risingwave`'s dialect is a genuine subclass of SQLAlchemy's own +Postgres dialect (via psycopg2), so DDL/pagination compile with standard +Postgres semantics -- no ClickHouse-style mandatory table option needed. +""" + +import re +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, + text, +) +from sqlalchemy.engine import Connection, Engine + +from superset.db_engine_specs.risingwave import RisingWaveDbEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("sqlalchemy_risingwave") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 4566 + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("risingwavelabs/risingwave") + container.with_exposed_ports(PORT) + container.with_command("playground") + # The actual startup banner reads "RisingWave standalone mode is + # ready." -- confirmed against a real container's logs. + container.waiting_for( + LogMessageWaitStrategy(re.compile("RisingWave standalone mode is ready")) + ) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine(f"risingwave://root@{host}:{port}/dev") + + +def _flush(conn: Connection) -> None: + # RisingWave's storage engine checkpoints asynchronously: without an + # explicit FLUSH, a SELECT immediately after INSERT can see zero rows + # -- confirmed against a real instance (a bare INSERT commits fine, but + # the data isn't visible to a subsequent query until flushed). + conn.execute(text("FLUSH")) + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine, after_insert=_flush) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + RisingWaveDbEngineSpec.get_columns wraps a real SQLAlchemy Inspector; + this exercises that against actual server-reported column metadata + rather than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = RisingWaveDbEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = RisingWaveDbEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_vertica.py b/tests/testcontainers/db_engine_specs/test_vertica.py new file mode 100644 index 00000000000..dc446971a2e --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_vertica.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.vertica against a real Vertica instance, spun up on +demand via testcontainers. Run via .github/workflows/testcontainers.yml, +on the nightly cron / manual dispatch only (see `nightly_only: true` on +this dialect's matrix entry) -- Vertica Community Edition has a +well-documented ~12GB RAM floor to even start (a license-tier check baked +into the image), well above what a per-PR CI runner should be expected to +provide. + +VerticaEngineSpec extends PostgresBaseEngineSpec, and +sqla_vertica_python.vertica_python.VerticaDialect is a direct subclass of +SQLAlchemy's own postgresql.PGDialect with only an index-syntax override +-- no custom DDL or LIMIT/OFFSET compiler, so a bare +Column(..., primary_key=True) table and standard LIMIT/OFFSET pagination +both compile with plain Postgres semantics (Vertica auto-creates a +default superprojection; no explicit projection/segmentation clause is +needed the way ClickHouse needs an ENGINE=). + +Not verified locally in this environment: Vertica CE's RAM/image weight +was judged not worth pulling on a resource-constrained local machine per +session guidance -- CI-only verification, matching the nightly_only +gating. +""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.vertica import VerticaEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("vertica_python") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +PORT = 5433 + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("vertica/vertica-ce") + container.with_exposed_ports(PORT) + container.waiting_for(LogMessageWaitStrategy("Vertica is now running")) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(PORT) + yield create_engine(f"vertica+vertica_python://dbadmin@{host}:{port}/VMart") + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + VerticaEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = VerticaEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = VerticaEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer) diff --git a/tests/testcontainers/db_engine_specs/test_ydb.py b/tests/testcontainers/db_engine_specs/test_ydb.py new file mode 100644 index 00000000000..8e0d72a7a54 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_ydb.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests db_engine_specs.ydb against a real YDB instance, spun up on demand +via testcontainers. Run via .github/workflows/testcontainers.yml. + +YDB has no dedicated testcontainers module, so this uses a generic +DockerContainer against the official `ydbplatform/local-ydb` image, which +needs no auth for local/anonymous access -- YDBEngineSpec's own +`sqlalchemy_uri_placeholder` ("ydb://{host}:{port}/{database_name}") has +no username/password at all. +""" + +import time +from collections.abc import Iterator + +import pytest +from sqlalchemy import ( + Column, + create_engine, + inspect, + Integer, + MetaData, + Table as SATable, +) +from sqlalchemy.engine import Engine + +from superset.db_engine_specs.ydb import YDBEngineSpec +from superset.sql.parse import Table +from superset.utils.core import GenericDataType + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") +require_driver("ydb_sqlalchemy") + +from testcontainers.core.container import DockerContainer # noqa: E402 +from testcontainers.core.wait_strategies import PortWaitStrategy # noqa: E402 + +from ._pagination import ( # noqa: E402 + assert_paginated_query_returns_correct_rows_in_order, +) + +GRPC_PORT = 2136 +DATABASE = "/local" + + +@pytest.fixture(scope="module") +def engine() -> Iterator[Engine]: + container = DockerContainer("ydbplatform/local-ydb") + container.with_exposed_ports(GRPC_PORT) + # YDB's gRPC client does endpoint discovery: it asks the server for its + # "real" endpoints and reconnects to whatever comes back, rather than + # just using the address it was originally given. By default that's + # the container's own internal Docker hostname (e.g. "6abbb4bb0ab7"), + # which isn't reachable from the host. Binding the same port number on + # the host as inside the container, plus advertising "localhost" as + # the container's own hostname, makes the discovered endpoint + # ("localhost:2136") actually resolve to something reachable. + container.with_bind_ports(GRPC_PORT, GRPC_PORT) + container.with_kwargs(hostname="localhost") + container.with_env("YDB_USE_IN_MEMORY_PDISKS", "true") + container.waiting_for(PortWaitStrategy(GRPC_PORT)) + + with container: + host = container.get_container_host_ip() + port = container.get_exposed_port(GRPC_PORT) + eng = create_engine(f"yql://{host}:{port}{DATABASE}") + + # The gRPC port opens, and even a bare SELECT succeeds, before YDB's + # storage pools are fully initialized -- an actual CREATE TABLE can + # still fail with "database doesn't have storage pools at all to + # create tablet channels" (confirmed on a real instance). Probe with + # metadata.create_all()/drop_all() specifically, the same call the + # real tests below make: a raw `text("CREATE TABLE ...")` hits a + # separate, unrelated error ("Scheme operations cannot be executed + # inside transaction") that create_all()'s own DDL execution path + # doesn't, even with AUTOCOMMIT set on a manually-opened connection. + probe_metadata = MetaData() + SATable("pilot_ready", probe_metadata, Column("id", Integer, primary_key=True)) + last_error: Exception | None = None + for _ in range(30): + try: + probe_metadata.create_all(eng) + probe_metadata.drop_all(eng) + break + except Exception as ex: # noqa: BLE001 -- retry on any not-ready-yet error + last_error = ex + time.sleep(2) + else: + raise RuntimeError( + "YDB never became ready to create and use a table" + ) from last_error + + yield eng + + +def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: + """ + A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against + a real instance. Mocked tests cannot catch a dialect compiling this + incorrectly (see apache/superset#42899, where Trino emitted OFFSET + before LIMIT) -- only real execution can. + """ + assert_paginated_query_returns_correct_rows_in_order(engine) + + +def test_get_columns_maps_native_types(engine: Engine) -> None: + """ + YDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this + exercises that against actual server-reported column metadata rather + than a mocked Inspector. + """ + metadata = MetaData() + SATable( + "pilot_types", + metadata, + Column("id", Integer, primary_key=True), + Column("amount", Integer), + ) + metadata.create_all(engine) + + inspector = inspect(engine) + columns = YDBEngineSpec.get_columns(inspector, Table("pilot_types")) + + by_name = {col["column_name"]: col for col in columns} + assert set(by_name) == {"id", "amount"} + for col in by_name.values(): + spec = YDBEngineSpec.get_column_spec(str(col["type"])) + assert spec is not None + assert spec.generic_type == GenericDataType.NUMERIC + assert isinstance(spec.sqla_type, Integer)