diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index 2dff522e69d..5c032a9772d 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -99,16 +99,11 @@ jobs: - dialect: ydb timeout: 10 # OceanBase bootstraps a distributed-style cluster even in - # single-node MODE=MINI, and Vertica Community Edition has a - # well-documented ~12GB RAM floor to even start -- both too heavy - # for every PR's CI budget, so both run on the nightly cron / - # manual dispatch only. + # single-node MODE=MINI -- too heavy for every PR's CI budget, so + # it runs on the nightly cron / manual dispatch only. - dialect: oceanbase timeout: 20 nightly_only: true - - dialect: vertica - timeout: 20 - nightly_only: true timeout-minutes: ${{ matrix.timeout }} env: PYTHONPATH: ${{ github.workspace }} @@ -139,11 +134,17 @@ jobs: # oceanbase_py pins sqlalchemy-utils>=0.38.3,<0.39, which conflicts # outright with Superset's own sqlalchemy-utils==0.42.1 pin -- kept # out of the baseline dev install for the same reason as db2 above. - # --no-deps sidesteps that pin entirely: this job only needs - # oceanbase_py's dialect module importable, not its sqlalchemy-utils - # dependency satisfied, since nothing here calls into it. + # Installed as its own standalone package (not via `-e .[oceanbase]`) + # so --no-deps only skips *oceanbase_py's* dependencies -- applied + # to `-e .[oceanbase]` instead, --no-deps blocks pip from installing + # anything the extras marker pulls in, including oceanbase_py + # itself, which "succeeds" without actually installing it + # (confirmed on real CI: the install step reported success, but the + # module was still missing). This job only needs oceanbase_py's + # dialect module importable, not its sqlalchemy-utils dependency + # satisfied, since nothing here calls into it. if: matrix.dialect == 'oceanbase' - run: uv pip install --system --no-deps -e .[oceanbase] + run: uv pip install --system --no-deps "oceanbase_py>=0.0.1.2" - name: Install Firebird client library (libfbclient2) # sqlalchemy-firebird's driver (firebird-driver) is a pure-Python # ctypes wrapper (its wheel is py3-none-any) that dynamically loads diff --git a/requirements/development.in b/requirements/development.in index 2766bf35ab0..d75496be099 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # --e .[development,bigquery,clickhouse,cockroachdb,crate,databend,druid,duckdb,elasticsearch,fastmcp,firebird,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,risingwave,starrocks,trino,thumbnails,vertica,ydb] +-e .[development,bigquery,clickhouse,cockroachdb,crate,databend,druid,duckdb,elasticsearch,fastmcp,firebird,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,risingwave,starrocks,trino,thumbnails,ydb] -e ./superset-extensions-cli[test] # testcontainers-backed db_engine_specs tests (tests/testcontainers/) -- # see .github/workflows/testcontainers.yml @@ -40,8 +40,8 @@ # pulled in above via the clickhouse extra, is all the test needs), and # StarRocks has no dedicated testcontainers module at all -- its test uses # a generic DockerContainer plus the same mysqlclient the mysql extra -# already provides. databend/risingwave/firebird/ydb/vertica are the same -# story: none has a dedicated testcontainers module, so each test uses a -# generic DockerContainer plus whatever driver its own extra above already +# already provides. databend/risingwave/firebird/ydb are the same story: +# none has a dedicated testcontainers module, so each test uses a generic +# DockerContainer plus whatever driver its own extra above already # provides. testcontainers[cockroachdb,cratedb,mongodb,mssql,mysql,oracle,postgres,trino]>=4.15.0,<5 diff --git a/requirements/development.txt b/requirements/development.txt index ad10b6fec99..892f3797a2c 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -921,7 +921,6 @@ python-dateutil==2.9.0.post0 # pyhive # shillelagh # trino - # vertica-python python-dotenv==1.2.2 # via # -c requirements/base-constraint.txt @@ -1038,7 +1037,6 @@ six==1.17.0 # prison # python-dateutil # rfc3339-validator - # vertica-python # wtforms-json slack-sdk==3.43.0 # via @@ -1094,8 +1092,6 @@ sqlalchemy-utils==0.42.1 # apache-superset # apache-superset-core # flask-appbuilder -sqlalchemy-vertica-python==0.6.3 - # via apache-superset sqlglot==30.17.0 # via # -c requirements/base-constraint.txt @@ -1209,8 +1205,6 @@ verlib2==0.3.2 # via # crate # sqlalchemy-cratedb -vertica-python==1.4.0 - # via sqlalchemy-vertica-python vine==5.1.0 # via # -c requirements/base-constraint.txt diff --git a/tests/testcontainers/db_engine_specs/test_vertica.py b/tests/testcontainers/db_engine_specs/test_vertica.py deleted file mode 100644 index dc446971a2e..00000000000 --- a/tests/testcontainers/db_engine_specs/test_vertica.py +++ /dev/null @@ -1,121 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Tests db_engine_specs.vertica against a real Vertica instance, spun up on -demand via testcontainers. Run via .github/workflows/testcontainers.yml, -on the nightly cron / manual dispatch only (see `nightly_only: true` on -this dialect's matrix entry) -- Vertica Community Edition has a -well-documented ~12GB RAM floor to even start (a license-tier check baked -into the image), well above what a per-PR CI runner should be expected to -provide. - -VerticaEngineSpec extends PostgresBaseEngineSpec, and -sqla_vertica_python.vertica_python.VerticaDialect is a direct subclass of -SQLAlchemy's own postgresql.PGDialect with only an index-syntax override --- no custom DDL or LIMIT/OFFSET compiler, so a bare -Column(..., primary_key=True) table and standard LIMIT/OFFSET pagination -both compile with plain Postgres semantics (Vertica auto-creates a -default superprojection; no explicit projection/segmentation clause is -needed the way ClickHouse needs an ENGINE=). - -Not verified locally in this environment: Vertica CE's RAM/image weight -was judged not worth pulling on a resource-constrained local machine per -session guidance -- CI-only verification, matching the nightly_only -gating. -""" - -from collections.abc import Iterator - -import pytest -from sqlalchemy import ( - Column, - create_engine, - inspect, - Integer, - MetaData, - Table as SATable, -) -from sqlalchemy.engine import Engine - -from superset.db_engine_specs.vertica import VerticaEngineSpec -from superset.sql.parse import Table -from superset.utils.core import GenericDataType - -pytestmark = pytest.mark.testcontainers - -from ._driver import require_driver # noqa: E402 - -require_driver("testcontainers.core.container") -require_driver("vertica_python") - -from testcontainers.core.container import DockerContainer # noqa: E402 -from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402 - -from ._pagination import ( # noqa: E402 - assert_paginated_query_returns_correct_rows_in_order, -) - -PORT = 5433 - - -@pytest.fixture(scope="module") -def engine() -> Iterator[Engine]: - container = DockerContainer("vertica/vertica-ce") - container.with_exposed_ports(PORT) - container.waiting_for(LogMessageWaitStrategy("Vertica is now running")) - - with container: - host = container.get_container_host_ip() - port = container.get_exposed_port(PORT) - yield create_engine(f"vertica+vertica_python://dbadmin@{host}:{port}/VMart") - - -def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None: - """ - A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against - a real instance. Mocked tests cannot catch a dialect compiling this - incorrectly (see apache/superset#42899, where Trino emitted OFFSET - before LIMIT) -- only real execution can. - """ - assert_paginated_query_returns_correct_rows_in_order(engine) - - -def test_get_columns_maps_native_types(engine: Engine) -> None: - """ - VerticaEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this - exercises that against actual server-reported column metadata rather - than a mocked Inspector. - """ - metadata = MetaData() - SATable( - "pilot_types", - metadata, - Column("id", Integer, primary_key=True), - Column("amount", Integer), - ) - metadata.create_all(engine) - - inspector = inspect(engine) - columns = VerticaEngineSpec.get_columns(inspector, Table("pilot_types")) - - by_name = {col["column_name"]: col for col in columns} - assert set(by_name) == {"id", "amount"} - for col in by_name.values(): - spec = VerticaEngineSpec.get_column_spec(str(col["type"])) - assert spec is not None - assert spec.generic_type == GenericDataType.NUMERIC - assert isinstance(spec.sqla_type, Integer)