mirror of
https://github.com/apache/superset.git
synced 2026-09-05 23:12:01 +00:00
Superset Dev
fde85be3ea
Adds five more dialects to the testcontainers suite, stacked on top of the 7-dialect pilot in feat/testcontainers-nightly-pilot: mariadb, timescaledb, yugabytedb, monetdb, mongodb. mariadb, timescaledb, and yugabytedb are all wire-compatible with an existing base dialect (MySQL and Postgres respectively), so they reuse testcontainers' MySqlContainer/PostgresContainer classes pointed at a different image rather than needing new container-class wiring. yugabytedb specifically cannot reuse PostgresContainer's built-in readiness check, though: that execs `psql`, which the yugabyte image doesn't ship (only its own `ysqlsh`) -- uses a generic DockerContainer instead, started via `yugabyted start` and waiting on its own final startup log line. monetdb has no native testcontainers module; uses a generic DockerContainer with the documented MDB_* environment variables. Publishes an amd64-only image (confirmed running under Rosetta/QEMU emulation on Apple Silicon, unlike CrateDB's harder x86-64-v3 CPU requirement). mongodb needed a different data-setup approach, like elasticsearch before it: documents get inserted via the native pymongo driver, not SQL INSERT, since MongoDB is schemaless and Superset talks to it through pymongosql (a SQL-to-MongoDB translation layer requiring a `?mode=superset` query param). Two real bugs surfaced writing this one: testcontainers' MongoDbContainer.get_connection_url() has no database path or query string at all, so naively appending "&mode=superset" glued directly onto the port number instead of starting a query string; and the root user MongoDbContainer creates lives in the `admin` database, so connecting with a different default database in the URL requires authSource=admin or authentication fails outright. Confirmed pymongosql supports OFFSET (maps to MongoDB's native `skip`), unlike Elasticsearch's SQL layer. mariadb could not be verified locally in this environment: mysqlclient (MySQLdb) has a pre-existing, unrelated native-library linking issue against this machine's Homebrew-installed libmysqlclient. CI installs it via apt on Linux, where this does not occur -- same accepted pattern already used for crate/mssql/db2 in the base branch.
105 lines
4.4 KiB
Python
105 lines
4.4 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
"""
|
|
Tests db_engine_specs.mongodb against a real MongoDB instance, spun up on
|
|
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
|
|
|
MongoDB is schemaless, and Superset talks to it via `pymongosql`, a
|
|
SQL-to-MongoDB translation layer (dialect requires a `?mode=superset` query
|
|
param -- not part of testcontainers' own MongoDbContainer.get_connection_url()).
|
|
Documents get inserted via the native pymongo driver, not SQL INSERT,
|
|
matching how Superset actually encounters MongoDB in practice and avoiding
|
|
any assumption about pymongosql's own INSERT/DDL support.
|
|
"""
|
|
|
|
from collections.abc import Iterator
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, inspect, text
|
|
from sqlalchemy.engine import Engine
|
|
|
|
from superset.db_engine_specs.mongodb import MongoDBEngineSpec
|
|
from superset.sql.parse import Table
|
|
|
|
pytestmark = pytest.mark.testcontainers
|
|
|
|
from ._driver import require_driver # noqa: E402
|
|
|
|
require_driver("testcontainers.community.mongodb")
|
|
require_driver("pymongosql")
|
|
|
|
from testcontainers.community.mongodb import MongoDbContainer # noqa: E402
|
|
|
|
COLLECTION = "pilot_pagination"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def engine() -> Iterator[Engine]:
|
|
with MongoDbContainer("mongo:7.0.7") as container:
|
|
client = container.get_connection_client()
|
|
client[container.dbname][COLLECTION].insert_many([{"id": i} for i in range(10)])
|
|
# MongoDbContainer.get_connection_url() has no database path segment
|
|
# or query string at all (it only builds user:pass@host:port), so
|
|
# naively appending "&mode=superset" glues it straight onto the port
|
|
# number instead of starting a query string. Build the full URL
|
|
# ourselves instead of relying on string concatenation.
|
|
host = container.get_container_host_ip()
|
|
port = container.get_exposed_port(container.port)
|
|
# authSource=admin is required: MongoDbContainer creates its root
|
|
# user via MONGO_INITDB_ROOT_USERNAME, which lives in the `admin`
|
|
# database, not in `dbname` -- without it, auth fails against
|
|
# whatever database is in the URL path.
|
|
yield create_engine(
|
|
f"mongodb://{container.username}:{container.password}@{host}:{port}"
|
|
f"/{container.dbname}?mode=superset&authSource=admin"
|
|
)
|
|
|
|
|
|
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
|
"""
|
|
A plain LIMIT/OFFSET query, compiled and executed against a real
|
|
instance. Mocked tests cannot catch a dialect compiling this
|
|
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
|
before LIMIT) -- only real execution can. Unlike Elasticsearch's SQL
|
|
layer (which has no OFFSET support at all), pymongosql maps OFFSET to
|
|
MongoDB's native `skip`, so this dialect supports it.
|
|
"""
|
|
with engine.connect() as conn:
|
|
rows = conn.execute(
|
|
text(
|
|
f"SELECT id FROM {COLLECTION} ORDER BY id LIMIT 3 OFFSET 4" # noqa: S608
|
|
)
|
|
).fetchall()
|
|
|
|
assert [row.id for row in rows] == [4, 5, 6]
|
|
|
|
|
|
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
|
"""
|
|
MongoDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector, which
|
|
pymongosql implements by sampling real documents to infer column types
|
|
-- this exercises that against an actual running instance rather than
|
|
a mocked Inspector.
|
|
"""
|
|
inspector = inspect(engine)
|
|
columns = MongoDBEngineSpec.get_columns(inspector, Table(COLLECTION))
|
|
|
|
by_name = {col["column_name"]: col for col in columns}
|
|
assert "id" in by_name
|
|
spec = MongoDBEngineSpec.get_column_spec(str(by_name["id"]["type"]))
|
|
assert spec is not None
|