mirror of
https://github.com/apache/superset.git
synced 2026-09-10 01:04:25 +00:00
Compare commits
@@ -41,7 +41,7 @@ jobs:
|
||||
|
||||
- name: Check for File Changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
uses: $/.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
# allowlist (only v8.1.0+ are, at apache/infrastructure-actions'
|
||||
# actions.yml). Needs an INFRA request before this can de-vendor too.
|
||||
- name: Set up chart-testing
|
||||
uses: $/.github/actions/chart-testing-action
|
||||
uses: ./.github/actions/chart-testing-action
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
# db_engine_specs tests against real databases (testcontainers)
|
||||
name: Testcontainers
|
||||
|
||||
# Spins up real Docker containers (see tests/testcontainers/ for the current
|
||||
# dialect list) via testcontainers-python, which catches real dialect/driver
|
||||
# regressions -- the kind mocked db_engine_specs unit tests structurally
|
||||
# cannot, e.g. apache/superset#42899 (Trino emitting OFFSET before LIMIT).
|
||||
# Runs on a nightly cron (catches drift from a driver's own releases, not
|
||||
# just from Superset's changes) and on pull_request, scoped via `paths` to
|
||||
# only PRs that actually touch this test suite or the workflow itself, so
|
||||
# unrelated PRs across the repo are never affected.
|
||||
#
|
||||
# A matrix entry can set `nightly_only: true` to run only on the cron (or a
|
||||
# manual workflow_dispatch), never on pull_request -- for a dialect whose
|
||||
# image is too heavy (a multi-service cluster, a many-GB image, a slow
|
||||
# licensed installer) to justify adding its wall-clock/resource cost to
|
||||
# every PR that merely touches this suite. Omit the field entirely for a
|
||||
# normal dialect; it isn't nightly-only by default.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 5 * * *"
|
||||
workflow_dispatch: {}
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/testcontainers.yml"
|
||||
- "tests/testcontainers/**"
|
||||
- "superset/db_engine_specs/**"
|
||||
- "pyproject.toml"
|
||||
- "requirements/development.in"
|
||||
- "requirements/development.txt"
|
||||
|
||||
concurrency:
|
||||
# Scoped by ref, not just workflow name -- otherwise every PR run and the
|
||||
# nightly cron share one group, and starting the workflow on another PR
|
||||
# (or the nightly firing mid-PR-run) cancels an unrelated in-progress run.
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
compute-matrix:
|
||||
# Filters out `nightly_only` dialects for a pull_request run *before* any
|
||||
# testcontainers job is created for them, so a heavy dialect costs a PR
|
||||
# nothing at all -- not even its checkout/setup/driver-install steps --
|
||||
# rather than being merely skipped at the test step. A job-level `if:`
|
||||
# can't reference `matrix` (only github/inputs/needs/vars are available
|
||||
# there), so the filtering has to happen here, before the matrix exists,
|
||||
# instead of on the testcontainers job itself.
|
||||
runs-on: ubuntu-26.04
|
||||
outputs:
|
||||
include: ${{ steps.filter.outputs.include }}
|
||||
steps:
|
||||
- name: Filter nightly-only dialects
|
||||
id: filter
|
||||
env:
|
||||
# One entry per dialect rather than one job for the whole suite: a
|
||||
# single slow container would otherwise inflate the wall-clock
|
||||
# time for every dialect, not just its own. Running in parallel
|
||||
# means the suite's total time is bounded by the slowest dialect,
|
||||
# not the sum of all of them. Db2's first-boot init is documented
|
||||
# upstream as notably slow (a real instance bring-up, not just a
|
||||
# process start) and untested locally here (no arm64 image), so
|
||||
# it gets a wider timeout margin than the rest until real CI data
|
||||
# says otherwise.
|
||||
FULL_MATRIX: |
|
||||
[
|
||||
{"dialect": "cockroachdb", "timeout": 10},
|
||||
{"dialect": "crate", "timeout": 10},
|
||||
{"dialect": "trino", "timeout": 10},
|
||||
{"dialect": "mssql", "timeout": 10},
|
||||
{"dialect": "elasticsearch", "timeout": 10},
|
||||
{"dialect": "oracle", "timeout": 15},
|
||||
{"dialect": "db2", "timeout": 25},
|
||||
{"dialect": "mariadb", "timeout": 10},
|
||||
{"dialect": "timescaledb", "timeout": 10},
|
||||
{"dialect": "yugabytedb", "timeout": 10},
|
||||
{"dialect": "monetdb", "timeout": 10},
|
||||
{"dialect": "mongodb", "timeout": 10},
|
||||
{"dialect": "postgres", "timeout": 10},
|
||||
{"dialect": "mysql", "timeout": 10},
|
||||
{"dialect": "clickhouse", "timeout": 10},
|
||||
{"dialect": "starrocks", "timeout": 15},
|
||||
{"dialect": "databend", "timeout": 10},
|
||||
{"dialect": "risingwave", "timeout": 10},
|
||||
{"dialect": "firebird", "timeout": 10},
|
||||
{"dialect": "ydb", "timeout": 10},
|
||||
{"dialect": "oceanbase", "timeout": 20, "nightly_only": true}
|
||||
]
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
include="$(jq -c '.' <<<"$FULL_MATRIX")"
|
||||
else
|
||||
include="$(jq -c '[.[] | select(.nightly_only != true)]' <<<"$FULL_MATRIX")"
|
||||
fi
|
||||
echo "include=${include}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
testcontainers:
|
||||
needs: [compute-matrix]
|
||||
runs-on: ubuntu-26.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include: ${{ fromJson(needs.compute-matrix.outputs.include) }}
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_TESTENV: true
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
# This job's matrix installs exactly one dialect's testcontainers
|
||||
# driver for exactly this job, so treat that driver as required: a
|
||||
# broken/missing import should fail the job, not silently skip to a
|
||||
# misleadingly green, zero-tests-run result. See _driver.py.
|
||||
SUPERSET_TESTCONTAINERS_STRICT: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
with:
|
||||
python-version: current
|
||||
- name: Install db2 driver (ibm-db-sa)
|
||||
# ibm-db (the db2 DBAPI) ships no Linux arm64 wheel, so it's kept out
|
||||
# of the baseline dev install (requirements/development.in) to avoid
|
||||
# breaking the multi-platform dev Docker image build. Install it here
|
||||
# instead, only for this leg of the matrix.
|
||||
if: matrix.dialect == 'db2'
|
||||
run: uv pip install --system -e .[db2]
|
||||
- name: Install oceanbase driver (oceanbase_py)
|
||||
# oceanbase_py pins sqlalchemy-utils>=0.38.3,<0.39, which conflicts
|
||||
# outright with Superset's own sqlalchemy-utils==0.42.1 pin -- kept
|
||||
# out of the baseline dev install for the same reason as db2 above.
|
||||
# Installed as its own standalone package (not via `-e .[oceanbase]`)
|
||||
# so --no-deps only skips *oceanbase_py's* dependencies -- applied
|
||||
# to `-e .[oceanbase]` instead, --no-deps blocks pip from installing
|
||||
# anything the extras marker pulls in, including oceanbase_py
|
||||
# itself, which "succeeds" without actually installing it
|
||||
# (confirmed on real CI: the install step reported success, but the
|
||||
# module was still missing). This job only needs oceanbase_py's
|
||||
# dialect module importable, not its sqlalchemy-utils dependency
|
||||
# satisfied, since nothing here calls into it.
|
||||
if: >-
|
||||
matrix.dialect == 'oceanbase' &&
|
||||
(matrix.nightly_only != true ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch')
|
||||
run: uv pip install --system --no-deps "oceanbase_py>=0.0.1.2"
|
||||
- name: Install Firebird client library (libfbclient2)
|
||||
# sqlalchemy-firebird's driver (firebird-driver) is a pure-Python
|
||||
# ctypes wrapper (its wheel is py3-none-any) that dynamically loads
|
||||
# the native Firebird client library from the host at import time
|
||||
# -- it doesn't bundle that library itself, so it has to come from
|
||||
# the system package manager, only for this leg of the matrix.
|
||||
if: matrix.dialect == 'firebird'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfbclient2
|
||||
- name: Run testcontainers db_engine_specs tests (${{ matrix.dialect }})
|
||||
# nightly_only dialects are already excluded from the matrix itself
|
||||
# on pull_request runs (see the compute-matrix job above), so this
|
||||
# step needs no additional gating.
|
||||
run: |
|
||||
pytest --durations-min=2 -v -m testcontainers \
|
||||
./tests/testcontainers/db_engine_specs/test_${{ matrix.dialect }}.py \
|
||||
--junit-xml=test-results/junit-testcontainers-${{ matrix.dialect }}.xml
|
||||
- name: Upload JUnit test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: junit-results-testcontainers-${{ matrix.dialect }}
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
|
||||
actions-timeline:
|
||||
needs: [testcontainers]
|
||||
if: always()
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
actions: read
|
||||
steps:
|
||||
- uses: Kesin11/actions-timeline@57fc93f20c6da7fbc14063c6d24a2a5627c799ad # v3.2.0
|
||||
+1
-1
@@ -232,7 +232,7 @@ unknown impact as zero. Chart and dashboard purge endpoints are unchanged.
|
||||
- The dashboard datasource-based visibility fallback now fails closed: a dashboard whose member charts’ datasources cannot be resolved (deleted datasource rows, missing `datasource_id`, or unsupported datasource types) is no longer accessible to users without explicit editor/viewer rights, and a dashboard composed of semantic-view charts now requires `datasource_access` on (at least one of) its semantic views or their parent semantic layer — previously any authenticated user could open such a dashboard’s shell. Because the fallback now considers every member chart rather than only table-backed ones, a user holding `datasource_access` on any single member datasource — including a semantic view or its parent layer — can open a mixed dashboard that previously denied them. Dashboards with no charts remain accessible, and dashboards with explicit viewers are unaffected. Conversely, holders of `all_datasource_access` now see every published no-viewer dashboard in the dashboard list — including chart-less ones previously hidden by the inner joins — matching what the object-level gate already allowed them to open.
|
||||
- Version restore (`POST /api/v1/{chart,dashboard,dataset}/<uuid>/versions/<version_uuid>/restore`) now refuses an **externally managed** entity (`is_managed_externally = True`) with HTTP 403, enforcing server-side what the docs already promised. Previously the refusal existed only in the browser, so an otherwise-authorized editor could restore such an entity by calling the endpoint directly and have the restore overwritten on the next external sync. Soft-delete recovery is deliberately unaffected — it changes visibility, not content.
|
||||
- `SAMPLES_ROW_LIMIT` is now the default for `/datasource/samples` requests without a valid explicit `per_page`, rather than a hard per-request ceiling; explicit limits are honored up to the existing global row-limit ceiling, matching `/chart/data` SAMPLES requests.
|
||||
- The `cockroachdb` extra (`pip install apache-superset[cockroachdb]`) now installs `sqlalchemy-cockroachdb` instead of the abandoned `cockroachdb` package, whose SQLAlchemy dialect could not be imported under SQLAlchemy 2.0. Existing environments with the old package installed should `pip uninstall cockroachdb && pip install sqlalchemy-cockroachdb` (or simply reinstall the extra) to restore CockroachDB connectivity.
|
||||
- The `cockroachdb` extra (`pip install apache-superset[cockroachdb]`) now installs `sqlalchemy-cockroachdb` instead of the abandoned `cockroachdb` package, whose SQLAlchemy dialect could not be imported under SQLAlchemy 2.0. Existing environments with the old package installed must `pip uninstall cockroachdb` before reinstalling the extra -- both packages register the same `cockroachdb` SQLAlchemy dialect entry point, so leaving the old one in place can still load the abandoned implementation.
|
||||
|
||||
### Native Value filter "Select all" always targets the whole column
|
||||
|
||||
|
||||
@@ -259,8 +259,8 @@ Superset automatically retries webhook deliveries on `429 Too Many Requests` and
|
||||
|
||||
### Kubernetes-specific
|
||||
|
||||
- You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override.
|
||||
- You can see the dedicated docs about [Kubernetes installation](/admin-docs/installation/kubernetes) for more details.
|
||||
- You must have a `celery beat` pod running. For Kubernetes deployments, use the [Apache Superset Kubernetes Operator documentation](https://apache.github.io/superset-kubernetes-operator/) for deployment configuration.
|
||||
- Existing legacy Helm chart deployments configure this with `supersetCeleryBeat.enabled = true` in the values override.
|
||||
|
||||
### Docker Compose specific
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ Since `docker compose` is primarily designed to run a set of containers on **a s
|
||||
and can't support requirements for **high availability**, we do not support nor recommend
|
||||
using our `docker compose` constructs to support production-type use-cases. For single host
|
||||
environments, we recommend using [minikube](https://minikube.sigs.k8s.io/docs/start/) along
|
||||
with our [installing on k8s](https://superset.apache.org/admin-docs/installation/running-on-kubernetes)
|
||||
with our [Kubernetes installation](/admin-docs/installation/kubernetes)
|
||||
documentation.
|
||||
:::
|
||||
|
||||
|
||||
@@ -33,13 +33,13 @@ Ideally you will build your own image of Superset that extends `lean`, adding wh
|
||||
|
||||
**Summary:** This is the best-practice way to deploy a production instance of Superset, but has the steepest skill requirement - someone who knows Kubernetes.
|
||||
|
||||
You will deploy Superset into a K8s cluster. The most common method is using the community-maintained Helm chart, though work is now underway to implement [SIP-149 - a Kubernetes Operator for Superset](https://github.com/apache/superset/issues/31408).
|
||||
You will deploy Superset into a K8s cluster. The recommended method is the official [Apache Superset Kubernetes Operator](https://apache.github.io/superset-kubernetes-operator/). The in-tree Helm chart is deprecated and is not recommended for new deployments.
|
||||
|
||||
A K8s deployment can scale up and down based on usage and deploy rolling updates with zero downtime - features that big deployments appreciate.
|
||||
|
||||
**Responsibilities**
|
||||
|
||||
You will need to build your own Docker image, and back up your metadata DB, both as described in Docker Compose above. You'll also need to customize your Helm chart values and deploy and maintain your Kubernetes cluster.
|
||||
You will need to build your own Docker image, and back up your metadata DB, both as described in Docker Compose above. You'll also need to configure the operator's Superset resources and deploy and maintain your Kubernetes cluster.
|
||||
|
||||
## [PyPI (Python)](/admin-docs/installation/pypi)
|
||||
|
||||
|
||||
@@ -13,499 +13,35 @@ import useBaseUrl from '@docusaurus/useBaseUrl';
|
||||
<br />
|
||||
<br />
|
||||
|
||||
Running Superset on Kubernetes is supported with the provided [Helm](https://helm.sh/) chart
|
||||
found in the official [Superset helm repository](https://apache.github.io/superset/index.yaml).
|
||||
Running Superset on Kubernetes is supported through the official
|
||||
[Apache Superset Kubernetes Operator](https://apache.github.io/superset-kubernetes-operator/).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A Kubernetes cluster
|
||||
- Helm installed
|
||||
|
||||
:::note
|
||||
For simpler, single host environments, we recommend using
|
||||
[minikube](https://minikube.sigs.k8s.io/docs/start/) which is easy to setup on many platforms
|
||||
and works fantastically well with the Helm chart referenced here.
|
||||
:::warning
|
||||
The Superset Helm chart under
|
||||
[`helm/superset`](https://github.com/apache/superset/tree/master/helm/superset)
|
||||
is deprecated and is not recommended for new Kubernetes deployments.
|
||||
:::
|
||||
|
||||
## Running
|
||||
Use the operator documentation as the source of truth for Kubernetes installation and operations.
|
||||
It covers installing the operator, defining Superset deployments with Kubernetes custom resources,
|
||||
configuring dependencies, upgrades, and operational workflows.
|
||||
|
||||
1. Add the Superset helm repository
|
||||
## New Deployments
|
||||
|
||||
```sh
|
||||
helm repo add superset https://apache.github.io/superset
|
||||
"superset" has been added to your repositories
|
||||
```
|
||||
Start with the
|
||||
[Apache Superset Kubernetes Operator documentation](https://apache.github.io/superset-kubernetes-operator/).
|
||||
|
||||
2. View charts in repo
|
||||
## Existing Helm Deployments
|
||||
|
||||
```sh
|
||||
helm search repo superset
|
||||
NAME CHART VERSION APP VERSION DESCRIPTION
|
||||
superset/superset 0.1.1 1.0 Apache Superset is a modern, enterprise-ready b...
|
||||
```
|
||||
Existing Helm chart users should plan a migration to the operator. Follow the
|
||||
[Helm chart migration guide](https://apache.github.io/superset-kubernetes-operator/user-guide/migration/)
|
||||
for the recommended migration path.
|
||||
|
||||
3. Configure your setting overrides
|
||||
Until migration is complete, legacy chart reference material remains available in the deprecated
|
||||
[Helm chart README](https://github.com/apache/superset/tree/master/helm/superset).
|
||||
|
||||
Just like any typical Helm chart, you'll need to craft a `values.yaml` file that would define/override any of the values exposed into the default [values.yaml](https://github.com/apache/superset/tree/master/helm/superset/values.yaml), or from any of the dependent charts it depends on:
|
||||
## Resources
|
||||
|
||||
- [bitnami/redis](https://artifacthub.io/packages/helm/bitnami/redis)
|
||||
- [bitnami/postgresql](https://artifacthub.io/packages/helm/bitnami/postgresql)
|
||||
|
||||
More info down below on some important overrides you might need.
|
||||
|
||||
4. Install and run
|
||||
|
||||
```sh
|
||||
helm upgrade --install --values my-values.yaml superset superset/superset
|
||||
```
|
||||
|
||||
You should see various pods popping up, such as:
|
||||
|
||||
```sh
|
||||
kubectl get pods
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
superset-celerybeat-7cdcc9575f-k6xmc 1/1 Running 0 119s
|
||||
superset-f5c9c667-dw9lp 1/1 Running 0 4m7s
|
||||
superset-f5c9c667-fk8bk 1/1 Running 0 4m11s
|
||||
superset-init-db-zlm9z 0/1 Completed 0 111s
|
||||
superset-postgresql-0 1/1 Running 0 6d20h
|
||||
superset-redis-master-0 1/1 Running 0 6d20h
|
||||
superset-worker-75b48bbcc-jmmjr 1/1 Running 0 4m8s
|
||||
superset-worker-75b48bbcc-qrq49 1/1 Running 0 4m12s
|
||||
```
|
||||
|
||||
The exact list will depend on some of your specific configuration overrides but you should generally expect:
|
||||
|
||||
- N `superset-xxxx-yyyy` and `superset-worker-xxxx-yyyy` pods (depending on your `supersetNode.replicaCount` and `supersetWorker.replicaCount` values)
|
||||
- 1 `superset-postgresql-0` depending on your postgres settings
|
||||
- 1 `superset-redis-master-0` depending on your redis settings
|
||||
- 1 `superset-celerybeat-xxxx-yyyy` pod if you have `supersetCeleryBeat.enabled = true` in your values overrides
|
||||
|
||||
1. Access it
|
||||
|
||||
The chart will publish appropriate services to expose the Superset UI internally within your k8s cluster. To access it externally you will have to either:
|
||||
|
||||
- Configure the Service as a `LoadBalancer` or `NodePort`
|
||||
- Set up an `Ingress` for it - the chart includes a definition, but will need to be tuned to your needs (hostname, tls, annotations etc...)
|
||||
- Set up a Gateway API `HTTPRoute` for it - see [Exposing Superset via Gateway API (HTTPRoute)](#exposing-superset-via-gateway-api-httproute) below
|
||||
- Run `kubectl port-forward superset-xxxx-yyyy :8088` to directly tunnel one pod's port into your localhost
|
||||
|
||||
Depending how you configured external access, the URL will vary. Once you've identified the appropriate URL you can log in with:
|
||||
|
||||
- user: `admin`
|
||||
- password: `admin`
|
||||
|
||||
## Important settings
|
||||
|
||||
### Security settings
|
||||
|
||||
Default security settings and passwords are included but you **MUST** update them to run `prod` instances, in particular:
|
||||
|
||||
```yaml
|
||||
postgresql:
|
||||
postgresqlPassword: superset
|
||||
```
|
||||
|
||||
Make sure, you set a unique strong complex alphanumeric string for your SECRET_KEY and use a tool to help you generate
|
||||
a sufficiently random sequence.
|
||||
|
||||
- To generate a good key you can run, `openssl rand -base64 42`
|
||||
|
||||
```yaml
|
||||
configOverrides:
|
||||
secret: |
|
||||
SECRET_KEY = 'YOUR_OWN_RANDOM_GENERATED_SECRET_KEY'
|
||||
```
|
||||
|
||||
If you want to change the previous secret key then you should rotate the keys.
|
||||
Default secret key for kubernetes deployment is `thisISaSECRET_1234`
|
||||
|
||||
```yaml
|
||||
configOverrides:
|
||||
my_override: |
|
||||
PREVIOUS_SECRET_KEY = 'YOUR_PREVIOUS_SECRET_KEY'
|
||||
SECRET_KEY = 'YOUR_OWN_RANDOM_GENERATED_SECRET_KEY'
|
||||
init:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
. {{ .Values.configMountPath }}/superset_bootstrap.sh
|
||||
superset re-encrypt-secrets
|
||||
. {{ .Values.configMountPath }}/superset_init.sh
|
||||
```
|
||||
|
||||
:::note
|
||||
Superset uses [Scarf Gateway](https://about.scarf.sh/) to collect telemetry data. Knowing the installation counts for different Superset versions informs the project's decisions about patching and long-term support. Scarf purges personally identifiable information (PII) and provides only aggregated statistics.
|
||||
|
||||
There are two independent telemetry channels:
|
||||
|
||||
- **Image pulls** (Scarf Gateway): to opt out, edit the `repository:` line in your `helm/superset/values.yaml` file, replacing `apachesuperset.docker.scarf.sh/apache/superset` with `apache/superset` to pull the image directly from Docker Hub.
|
||||
- **The analytics pixel** rendered in the UI: to opt out, set the `SCARF_ANALYTICS` environment variable to `false` on the Superset containers via `extraEnv` in your `values.yaml`:
|
||||
|
||||
```yaml
|
||||
extraEnv:
|
||||
SCARF_ANALYTICS: 'false'
|
||||
```
|
||||
|
||||
This is read at runtime, so it takes effect on the pre-built images without rebuilding the frontend.
|
||||
:::
|
||||
|
||||
### Dependencies
|
||||
|
||||
Install additional packages and do any other bootstrap configuration in the bootstrap script.
|
||||
For production clusters it's recommended to build own image with this step done in CI.
|
||||
|
||||
:::note
|
||||
|
||||
Superset requires a Python DB-API database driver and a SQLAlchemy
|
||||
dialect to be installed for each datastore you want to connect to.
|
||||
|
||||
See [Install Database Drivers](/user-docs/databases#installing-database-drivers) for more information.
|
||||
It is recommended that you refer to versions listed in
|
||||
[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml)
|
||||
instead of hard-coding them in your bootstrap script, as seen below.
|
||||
|
||||
:::
|
||||
|
||||
The following example installs the drivers for BigQuery and Elasticsearch, allowing you to connect to these data sources within your Superset setup:
|
||||
|
||||
```yaml
|
||||
bootstrapScript: |
|
||||
#!/bin/bash
|
||||
uv pip install .[postgres] \
|
||||
.[bigquery] \
|
||||
.[elasticsearch] &&\
|
||||
if [ ! -f ~/bootstrap ]; then echo "Running Superset with uid {{ .Values.runAsUser }}" > ~/bootstrap; fi
|
||||
```
|
||||
|
||||
### superset_config.py
|
||||
|
||||
The default `superset_config.py` is fairly minimal and you will very likely need to extend it. This is done by specifying one or more key/value entries in `configOverrides`, e.g.:
|
||||
|
||||
```yaml
|
||||
configOverrides:
|
||||
my_override: |
|
||||
# This will make sure the redirect_uri is properly computed, even with SSL offloading
|
||||
ENABLE_PROXY_FIX = True
|
||||
FEATURE_FLAGS = {
|
||||
"DYNAMIC_PLUGINS": True
|
||||
}
|
||||
```
|
||||
|
||||
Those will be evaluated as Helm templates and therefore will be able to reference other `values.yaml` variables e.g. `{{ .Values.ingress.hosts[0] }}` will resolve to your ingress external domain.
|
||||
|
||||
The entire `superset_config.py` will be installed as a secret, so it is safe to pass sensitive parameters directly... however it might be more readable to use secret env variables for that.
|
||||
|
||||
Full python files can be provided by running `helm upgrade --install --values my-values.yaml --set-file configOverrides.oauth=set_oauth.py`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Those can be passed as key/values either with `extraEnv` or `extraSecretEnv` if they're sensitive. They can then be referenced from `superset_config.py` using e.g. `os.environ.get("VAR")`.
|
||||
|
||||
```yaml
|
||||
extraEnv:
|
||||
SMTP_HOST: smtp.gmail.com
|
||||
SMTP_USER: user@gmail.com
|
||||
SMTP_PORT: '587'
|
||||
SMTP_MAIL_FROM: user@gmail.com
|
||||
|
||||
extraSecretEnv:
|
||||
SMTP_PASSWORD: xxxx
|
||||
|
||||
configOverrides:
|
||||
smtp: |
|
||||
import ast
|
||||
SMTP_HOST = os.getenv("SMTP_HOST","localhost")
|
||||
SMTP_STARTTLS = ast.literal_eval(os.getenv("SMTP_STARTTLS", "True"))
|
||||
SMTP_SSL = ast.literal_eval(os.getenv("SMTP_SSL", "False"))
|
||||
SMTP_USER = os.getenv("SMTP_USER","superset")
|
||||
SMTP_PORT = os.getenv("SMTP_PORT",25)
|
||||
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD","superset")
|
||||
```
|
||||
|
||||
### System packages
|
||||
|
||||
If new system packages are required, they can be installed before application startup by overriding the container's `command`, e.g.:
|
||||
|
||||
```yaml
|
||||
supersetWorker:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
apt update
|
||||
apt install -y somepackage
|
||||
apt autoremove -yqq --purge
|
||||
apt clean
|
||||
|
||||
# Run celery worker
|
||||
. {{ .Values.configMountPath }}/superset_bootstrap.sh; celery --app=superset.tasks.celery_app:app worker
|
||||
```
|
||||
|
||||
### Data sources
|
||||
|
||||
Data source definitions can be automatically declared by providing key/value yaml definitions in `extraConfigs`:
|
||||
|
||||
```yaml
|
||||
extraConfigs:
|
||||
import_datasources.yaml: |
|
||||
databases:
|
||||
- allow_file_upload: true
|
||||
allow_ctas: true
|
||||
allow_cvas: true
|
||||
database_name: example-db
|
||||
extra: "{\r\n \"metadata_params\": {},\r\n \"engine_params\": {},\r\n \"\
|
||||
metadata_cache_timeout\": {},\r\n \"schemas_allowed_for_file_upload\": []\r\n\
|
||||
}"
|
||||
sqlalchemy_uri: example://example-db.local
|
||||
tables: []
|
||||
```
|
||||
|
||||
Those will also be mounted as secrets and can include sensitive parameters.
|
||||
|
||||
## Configuration Examples
|
||||
|
||||
### Setting up OAuth
|
||||
|
||||
:::note
|
||||
|
||||
OAuth setup requires that the [authlib](https://authlib.org/) Python library is installed. This can
|
||||
be done using `pip` by updating the `bootstrapScript`. See the [Dependencies](#dependencies) section
|
||||
for more information.
|
||||
|
||||
:::
|
||||
|
||||
```yaml
|
||||
extraEnv:
|
||||
AUTH_DOMAIN: example.com
|
||||
|
||||
extraSecretEnv:
|
||||
GOOGLE_KEY: xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com
|
||||
GOOGLE_SECRET: xxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
configOverrides:
|
||||
enable_oauth: |
|
||||
# This will make sure the redirect_uri is properly computed, even with SSL offloading
|
||||
ENABLE_PROXY_FIX = True
|
||||
|
||||
from flask_appbuilder.security.manager import AUTH_OAUTH
|
||||
AUTH_TYPE = AUTH_OAUTH
|
||||
OAUTH_PROVIDERS = [
|
||||
{
|
||||
"name": "google",
|
||||
"icon": "fa-google",
|
||||
"token_key": "access_token",
|
||||
"remote_app": {
|
||||
"client_id": os.getenv("GOOGLE_KEY"),
|
||||
"client_secret": os.getenv("GOOGLE_SECRET"),
|
||||
"api_base_url": "https://www.googleapis.com/oauth2/v2/",
|
||||
"client_kwargs": {"scope": "email profile"},
|
||||
"request_token_url": None,
|
||||
"access_token_url": "https://accounts.google.com/o/oauth2/token",
|
||||
"authorize_url": "https://accounts.google.com/o/oauth2/auth",
|
||||
"authorize_params": {"hd": os.getenv("AUTH_DOMAIN", "")}
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
# Map Authlib roles to superset roles
|
||||
AUTH_ROLE_ADMIN = 'Admin'
|
||||
AUTH_ROLE_PUBLIC = 'Public'
|
||||
|
||||
# Will allow user self registration, allowing to create Flask users from Authorized User
|
||||
AUTH_USER_REGISTRATION = True
|
||||
|
||||
# The default user self registration role
|
||||
AUTH_USER_REGISTRATION_ROLE = "Admin"
|
||||
```
|
||||
|
||||
### Exposing Superset via Gateway API (HTTPRoute)
|
||||
|
||||
As an alternative to `Ingress`, the chart can create a [Gateway API](https://gateway-api.sigs.k8s.io/)
|
||||
`HTTPRoute` that attaches to a Gateway already running in your cluster. This requires the Gateway
|
||||
API CRDs serving the configured `httproute.apiVersion` (`gateway.networking.k8s.io/v1` by default)
|
||||
to be installed, along with a Gateway resource for the route to attach to. If the Gateway lives in
|
||||
a different namespace than the `HTTPRoute` (as in the
|
||||
example below), its listener's `allowedRoutes` must explicitly permit routes from this release's
|
||||
namespace, or the `HTTPRoute` will install successfully but never attach.
|
||||
|
||||
```yaml
|
||||
httproute:
|
||||
enabled: true
|
||||
parentRefs:
|
||||
- name: my-gateway
|
||||
namespace: gateway-system
|
||||
hostnames:
|
||||
- superset.example.com
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
```
|
||||
|
||||
- `httproute.parentRefs` lists the Gateway(s) the route attaches to.
|
||||
- `httproute.hostnames` matches against the HTTP `Host` header; it's templated, so values like
|
||||
`{{ .Release.Name }}` can be used.
|
||||
- `httproute.rules` are routing rules backed by the Superset service; each rule accepts standard
|
||||
`matches`, `filters`, and `timeouts` fields, and an optional `weight` (defaults to `1`) applied to
|
||||
its single backend reference. Since each rule maps to one backend, `weight` has no traffic-splitting
|
||||
effect here; it only matters if you fork the template to add multiple `backendRefs` to a rule.
|
||||
`timeouts` only joined the Gateway API Standard channel in v1.2, so it requires both v1.2+ CRDs
|
||||
and a supporting controller; drop it if either predates that.
|
||||
- If `supersetWebsockets.enabled` is set, an extra rule routing `supersetWebsockets.ingress.path`
|
||||
(default `/ws`) to the `-ws` service is appended automatically, mirroring the `Ingress` behavior.
|
||||
WebSocket upgrade support is controller-dependent under Gateway API; check your Gateway
|
||||
implementation's docs in case it needs an explicit protocol opt-in for global async queries to
|
||||
keep working behind a Gateway.
|
||||
- If `supersetMcp.enabled` and `supersetMcp.httproute.enabled` are both set, an extra rule routing
|
||||
`supersetMcp.httproute.path` to the `-mcp` service is appended as well. Don't expose this route
|
||||
without first enabling MCP authentication — see the
|
||||
[MCP Server Deployment & Authentication](/admin-docs/configuration/mcp-server#authentication) doc;
|
||||
by default the MCP server runs in dev mode with auth disabled.
|
||||
- Set `httproute.apiVersion` to `gateway.networking.k8s.io/v1beta1` if your cluster's Gateway API
|
||||
installation hasn't promoted `HTTPRoute` to `v1` yet.
|
||||
|
||||
### Enable Alerts and Reports
|
||||
|
||||
For this, as per the [Alerts and Reports doc](/admin-docs/configuration/alerts-reports), you will need to:
|
||||
|
||||
#### Install a supported webdriver in the Celery worker
|
||||
|
||||
This is done either by using a custom image that has the webdriver pre-installed, or installing at startup time by overriding the `command`. Here's a working example for `chromedriver`:
|
||||
|
||||
```yaml
|
||||
supersetWorker:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
# Install chrome webdriver
|
||||
# See https://github.com/apache/superset/blob/4fa3b6c7185629b87c27fc2c0e5435d458f7b73d/docs/src/pages/admin-docs/installation/email_reports.mdx
|
||||
apt-get update
|
||||
apt-get install -y wget
|
||||
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
|
||||
apt-get install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb
|
||||
wget https://chromedriver.storage.googleapis.com/88.0.4324.96/chromedriver_linux64.zip
|
||||
apt-get install -y zip
|
||||
unzip chromedriver_linux64.zip
|
||||
chmod +x chromedriver
|
||||
mv chromedriver /usr/bin
|
||||
apt-get autoremove -yqq --purge
|
||||
apt-get clean
|
||||
rm -f google-chrome-stable_current_amd64.deb chromedriver_linux64.zip
|
||||
|
||||
# Run
|
||||
. {{ .Values.configMountPath }}/superset_bootstrap.sh; celery --app=superset.tasks.celery_app:app worker
|
||||
```
|
||||
|
||||
#### Run the Celery beat
|
||||
|
||||
This pod will trigger the scheduled tasks configured in the alerts and reports UI section:
|
||||
|
||||
```yaml
|
||||
supersetCeleryBeat:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
#### Configure the appropriate Celery jobs and SMTP/Slack settings
|
||||
|
||||
```yaml
|
||||
extraEnv:
|
||||
SMTP_HOST: smtp.gmail.com
|
||||
SMTP_USER: user@gmail.com
|
||||
SMTP_PORT: '587'
|
||||
SMTP_MAIL_FROM: user@gmail.com
|
||||
|
||||
extraSecretEnv:
|
||||
SLACK_API_TOKEN: xoxb-xxxx-yyyy
|
||||
SMTP_PASSWORD: xxxx-yyyy
|
||||
|
||||
configOverrides:
|
||||
feature_flags: |
|
||||
import ast
|
||||
|
||||
FEATURE_FLAGS = {
|
||||
"ALERT_REPORTS": True
|
||||
}
|
||||
|
||||
SMTP_HOST = os.getenv("SMTP_HOST","localhost")
|
||||
SMTP_STARTTLS = ast.literal_eval(os.getenv("SMTP_STARTTLS", "True"))
|
||||
SMTP_SSL = ast.literal_eval(os.getenv("SMTP_SSL", "False"))
|
||||
SMTP_USER = os.getenv("SMTP_USER","superset")
|
||||
SMTP_PORT = os.getenv("SMTP_PORT",25)
|
||||
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD","superset")
|
||||
SMTP_MAIL_FROM = os.getenv("SMTP_MAIL_FROM","superset@superset.com")
|
||||
|
||||
SLACK_API_TOKEN = os.getenv("SLACK_API_TOKEN",None)
|
||||
celery_conf: |
|
||||
from celery.schedules import crontab
|
||||
|
||||
class CeleryConfig:
|
||||
broker_url = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
|
||||
imports = (
|
||||
"superset.sql_lab",
|
||||
"superset.tasks.cache",
|
||||
"superset.tasks.scheduler",
|
||||
)
|
||||
result_backend = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
|
||||
task_annotations = {
|
||||
"sql_lab.get_sql_results": {
|
||||
"rate_limit": "100/s",
|
||||
},
|
||||
}
|
||||
beat_schedule = {
|
||||
"reports.scheduler": {
|
||||
"task": "reports.scheduler",
|
||||
"schedule": crontab(minute="*", hour="*"),
|
||||
},
|
||||
"reports.prune_log": {
|
||||
"task": "reports.prune_log",
|
||||
'schedule': crontab(minute=0, hour=0),
|
||||
},
|
||||
'cache-warmup-hourly': {
|
||||
"task": "cache-warmup",
|
||||
"schedule": crontab(minute="*/30", hour="*"),
|
||||
"kwargs": {
|
||||
"strategy_name": "top_n_dashboards",
|
||||
"top_n": 10,
|
||||
"since": "7 days ago",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
CELERY_CONFIG = CeleryConfig
|
||||
reports: |
|
||||
EMAIL_PAGE_RENDER_WAIT = 60
|
||||
WEBDRIVER_BASEURL = "http://{{ template "superset.fullname" . }}:{{ .Values.service.port }}/"
|
||||
WEBDRIVER_BASEURL_USER_FRIENDLY = "https://www.example.com/"
|
||||
WEBDRIVER_TYPE= "chrome"
|
||||
WEBDRIVER_OPTION_ARGS = [
|
||||
"--force-device-scale-factor=2.0",
|
||||
"--high-dpi-support=2.0",
|
||||
"--headless",
|
||||
"--disable-gpu",
|
||||
"--disable-dev-shm-usage",
|
||||
# This is required because our process runs as root (in order to install pip packages)
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-extensions",
|
||||
]
|
||||
```
|
||||
|
||||
### Load the Examples data and dashboards
|
||||
|
||||
If you are trying Superset out and want some data and dashboards to explore, you can load some examples by creating a `my_values.yaml` and deploying it as described above in the **Configure your setting overrides** step of the **Running** section.
|
||||
To load the examples, add the following to the `my_values.yaml` file:
|
||||
|
||||
```yaml
|
||||
init:
|
||||
loadExamples: true
|
||||
```
|
||||
|
||||
:::resources
|
||||
|
||||
- [Tutorial: Mastering Data Visualization — Installing Superset on Kubernetes with Helm Chart](https://mahira-technology.medium.com/mastering-data-visualization-installing-superset-on-kubernetes-cluster-using-helm-chart-e4ec99199e1e)
|
||||
- [Tutorial: Installing Apache Superset in Kubernetes](https://aws.plainenglish.io/installing-apache-superset-in-kubernetes-1aec192ac495)
|
||||
:::
|
||||
- [Apache Superset Kubernetes Operator documentation](https://apache.github.io/superset-kubernetes-operator/)
|
||||
- [Apache Superset Kubernetes Operator repository](https://github.com/apache/superset-kubernetes-operator)
|
||||
- [Helm chart migration guide](https://apache.github.io/superset-kubernetes-operator/user-guide/migration/)
|
||||
|
||||
@@ -93,7 +93,7 @@ Look through the GitHub issues. Issues tagged with
|
||||
|
||||
Superset could always use better documentation,
|
||||
whether as part of the official Superset docs,
|
||||
in docstrings, `docs/*.rst` or even on the web as blog posts or
|
||||
in docstrings, Markdown files in `docs/`, or even on the web as blog posts or
|
||||
articles. See [Documentation](./howtos.md#contributing-to-documentation) for more details.
|
||||
|
||||
### Add Translations
|
||||
|
||||
+1
-1
@@ -414,7 +414,7 @@ This can be used, for example, to convert UTC time to local time.
|
||||
Superset uses [Scarf](https://about.scarf.sh/) by default to collect basic telemetry data upon installing and/or running Superset. This data helps the maintainers of Superset better understand which versions of Superset are being used, in order to prioritize patch/minor releases and security fixes.
|
||||
We use the [Scarf Gateway](https://docs.scarf.sh/gateway/) to sit in front of container registries, the [scarf-js](https://about.scarf.sh/package-sdks) package to track `npm` installations, and a Scarf pixel to gather anonymous analytics on Superset page views.
|
||||
Scarf purges PII and provides aggregated statistics. Superset users can easily opt out of analytics in various ways documented [here](https://docs.scarf.sh/gateway/#do-not-track) and [here](https://docs.scarf.sh/package-analytics/#as-a-user-of-a-package-using-scarf-js-how-can-i-opt-out-of-analytics).
|
||||
You can also opt out of the analytics pixel by setting the `SCARF_ANALYTICS` environment variable to `false`. This is read at runtime, so setting it on the Superset container (for example via `extraEnv` in the Helm chart, or `docker/.env` for Docker Compose) disables the pixel on the pre-built images without rebuilding the frontend. Note that this only disables the page-view pixel; the Scarf Gateway (container registry) and `scarf-js` (`npm`) channels are opted out separately, as described above.
|
||||
You can also opt out of the analytics pixel by setting the `SCARF_ANALYTICS` environment variable to `false`. This is read at runtime, so setting it on the Superset container (for example via your Kubernetes deployment configuration, or `docker/.env` for Docker Compose) disables the pixel on the pre-built images without rebuilding the frontend. Note that this only disables the page-view pixel; the Scarf Gateway (container registry) and `scarf-js` (`npm`) channels are opted out separately, as described above.
|
||||
Additional opt-out instructions are available on the [Docker Compose](/admin-docs/installation/docker-compose) and [Kubernetes](/admin-docs/installation/kubernetes) installation pages.
|
||||
|
||||
## Does Superset have an archive panel or trash bin from which a user can recover deleted assets?
|
||||
|
||||
+1
-1
@@ -260,7 +260,7 @@ Understanding the Superset Points of View
|
||||
- Deploying Superset
|
||||
|
||||
- [Official Docker image](https://hub.docker.com/r/apache/superset)
|
||||
- [Helm Chart](https://github.com/apache/superset/tree/master/helm/superset)
|
||||
- [Kubernetes Operator](https://apache.github.io/superset-kubernetes-operator/)
|
||||
|
||||
- Recordings of Past [Superset Community Events](https://preset.io/events)
|
||||
|
||||
|
||||
@@ -3215,7 +3215,7 @@
|
||||
"logo": "cockroachdb.png",
|
||||
"homepage_url": "https://www.cockroachlabs.com/",
|
||||
"categories": ["Traditional RDBMS", "Open Source"],
|
||||
"pypi_packages": ["cockroachdb"],
|
||||
"pypi_packages": ["sqlalchemy-cockroachdb", "psycopg2-binary"],
|
||||
"connection_string": "cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable",
|
||||
"default_port": 26257,
|
||||
"docs_url": "https://github.com/cockroachdb/sqlalchemy-cockroachdb",
|
||||
|
||||
Vendored
+1
-1
@@ -14,7 +14,7 @@ Superset is designed for data exploration and visualization at scale. It feature
|
||||
- [Architecture](https://superset.apache.org/docs/installation/architecture): Production deployment architecture and components
|
||||
- [Docker Compose](https://superset.apache.org/docs/installation/docker-compose): Install Superset using Docker Compose
|
||||
- [Docker Builds](https://superset.apache.org/docs/installation/docker-builds): Building and customizing Docker images
|
||||
- [Kubernetes](https://superset.apache.org/docs/installation/kubernetes): Deploy Superset on Kubernetes with Helm
|
||||
- [Kubernetes](https://superset.apache.org/docs/installation/kubernetes): Deploy Superset on Kubernetes with the official Kubernetes Operator
|
||||
- [PyPI](https://superset.apache.org/docs/installation/pypi): Install from PyPI using pip
|
||||
- [Upgrading Superset](https://superset.apache.org/docs/installation/upgrading-superset): Upgrade between Superset versions
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
apiVersion: v2
|
||||
deprecated: true
|
||||
appVersion: "6.1.0"
|
||||
description: Apache Superset is a modern, enterprise-ready business intelligence web application
|
||||
name: superset
|
||||
@@ -25,11 +26,7 @@ keywords:
|
||||
- data science
|
||||
sources:
|
||||
- https://github.com/apache/superset
|
||||
maintainers:
|
||||
- name: craig-rueda
|
||||
email: craig@craigrueda.com
|
||||
url: https://github.com/craig-rueda
|
||||
version: 0.22.7 # See [README](https://github.com/apache/superset/blob/master/helm/superset/README.md#versioning) for version details.
|
||||
version: 0.22.8 # See [README](https://github.com/apache/superset/blob/master/helm/superset/README.md#versioning) for version details.
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: 16.7.27
|
||||
|
||||
+10
-1
@@ -23,7 +23,9 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs
|
||||
|
||||
# superset
|
||||
|
||||
|
||||
> **:exclamation: This Helm Chart is deprecated!**
|
||||
|
||||
|
||||
|
||||
Apache Superset is a modern, enterprise-ready business intelligence web application
|
||||
|
||||
@@ -33,6 +35,13 @@ Apache Superset is a modern, enterprise-ready business intelligence web applicat
|
||||
|
||||
* <https://github.com/apache/superset>
|
||||
|
||||
## Deprecation Notice
|
||||
|
||||
> [!WARNING]
|
||||
> The Superset Helm chart is deprecated. For new Superset deployments on Kubernetes, use the official [Apache Superset Kubernetes Operator](https://github.com/apache/superset-kubernetes-operator) instead.
|
||||
>
|
||||
> Existing Helm chart users should plan a migration to the operator. Start with the [operator documentation](https://apache.github.io/superset-kubernetes-operator/) and the [Helm chart migration guide](https://apache.github.io/superset-kubernetes-operator/user-guide/migration/).
|
||||
|
||||
## TL;DR
|
||||
|
||||
```console
|
||||
|
||||
@@ -32,6 +32,13 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs
|
||||
|
||||
{{ template "chart.sourcesSection" . }}
|
||||
|
||||
## Deprecation Notice
|
||||
|
||||
> [!WARNING]
|
||||
> The Superset Helm chart is deprecated. For new Superset deployments on Kubernetes, use the official [Apache Superset Kubernetes Operator](https://github.com/apache/superset-kubernetes-operator) instead.
|
||||
>
|
||||
> Existing Helm chart users should plan a migration to the operator. Start with the [operator documentation](https://apache.github.io/superset-kubernetes-operator/) and the [Helm chart migration guide](https://apache.github.io/superset-kubernetes-operator/user-guide/migration/).
|
||||
|
||||
## TL;DR
|
||||
|
||||
```console
|
||||
|
||||
+25
-6
@@ -151,10 +151,9 @@ clickhouse = ["clickhouse-connect>=1.7.2, <2.0"]
|
||||
# 2.0). sqlalchemy-cockroachdb is the actively maintained replacement,
|
||||
# already linked from CockroachDbEngineSpec.metadata's docs_url, and
|
||||
# registers the same `cockroachdb` SQLAlchemy dialect entry point.
|
||||
# sqlalchemy-cockroachdb depends only on SQLAlchemy itself, not on a DBAPI
|
||||
# driver, so psycopg2-binary is pinned alongside it (matching the `postgres`
|
||||
# extra) to keep this extra self-contained -- CockroachDB speaks the
|
||||
# PostgreSQL wire protocol, so psycopg2 is what actually opens connections.
|
||||
# sqlalchemy-cockroachdb itself declares no DBAPI dependency (its own docs
|
||||
# require picking one), so pull in the same psycopg2-binary pin as the
|
||||
# `postgres` extra -- CockroachDB speaks the Postgres wire protocol.
|
||||
cockroachdb = ["sqlalchemy-cockroachdb>=2.0.0, <3", "psycopg2-binary==2.9.12"]
|
||||
crate = ["sqlalchemy-cratedb>=0.43.1, <1"]
|
||||
# sqlalchemy-d1's only release (0.1.0, Nov 2025) pins sqlalchemy<2,>=1.4,
|
||||
@@ -226,6 +225,11 @@ impala = ["impyla>=0.24.0, <0.25"]
|
||||
# superset/db_engine_specs/kusto.py's known_incompatibilities metadata.
|
||||
kusto = ["sqlalchemy-kusto>=3.1.2, <4"]
|
||||
kylin = ["kylinpy>=2.8.4, <2.9"]
|
||||
# MariaDB is a MySQL fork implementing the same wire protocol - connects via
|
||||
# the plain mysql dialect, same driver as mysql.
|
||||
mariadb = ["apache-superset[mysql]"]
|
||||
monetdb = ["sqlalchemy-monetdb>=2.1.0, <3", "pymonetdb>=1.9.1, <2"]
|
||||
mongodb = ["pymongosql>=0.7.3, <1"]
|
||||
mssql = ["pymssql>=2.3.13, <3"]
|
||||
# motherduck is an alias for duckdb - MotherDuck works via the duckdb driver
|
||||
motherduck = ["apache-superset[duckdb]"]
|
||||
@@ -274,18 +278,33 @@ tdengine = [
|
||||
"taos-ws-py>=0.7.0"
|
||||
]
|
||||
teradata = ["teradatasql>=20.0.0.67"]
|
||||
# TimescaleDB is a genuine Postgres extension, not a fork - connects via the
|
||||
# plain postgresql dialect, same driver as postgres.
|
||||
timescaledb = ["apache-superset[postgres]"]
|
||||
thumbnails = [] # deprecated, will be removed in 7.0
|
||||
vertica = ["sqlalchemy-vertica-python>= 0.6.3, < 0.7"]
|
||||
netezza = ["nzalchemy>= 11.1.2, < 11.2"]
|
||||
starrocks = ["starrocks>=1.3.4, <2"]
|
||||
doris = ["pydoris>=1.2.0, <2.0.0"]
|
||||
oceanbase = ["oceanbase_py>=0.0.1.2"]
|
||||
# No `oceanbase` extra is published here: oceanbase_py pins
|
||||
# sqlalchemy-utils>=0.38.3,<0.39, which conflicts outright with Superset's
|
||||
# own sqlalchemy-utils==0.42.1 pin, so `pip install apache-superset[oceanbase]`
|
||||
# could never resolve. CI installs oceanbase_py as a standalone package with
|
||||
# --no-deps instead (see testcontainers.yml). Add the extra back once
|
||||
# oceanbase_py relaxes that pin.
|
||||
ydb = ["ydb-sqlalchemy>=0.1.22", "ydb-sqlglot-plugin>=0.2.8"]
|
||||
# YugabyteDB's YSQL layer is fully Postgres-wire compatible - connects via
|
||||
# the plain postgresql dialect, same driver as postgres.
|
||||
yugabytedb = ["apache-superset[postgres]"]
|
||||
development = [
|
||||
# no bounds for apache-superset-extensions-cli until a stable version
|
||||
"apache-superset-extensions-cli",
|
||||
"boto3",
|
||||
"docker",
|
||||
# 7.0.0 raises `docker.errors.DockerException: ... Not supported URL
|
||||
# scheme http+docker` against the requests/urllib3 versions pinned
|
||||
# elsewhere in this file -- breaks testcontainers (tests/testcontainers/)
|
||||
# before any container even starts. 7.2.0 is confirmed working.
|
||||
"docker>=7.2.0",
|
||||
"flask-testing",
|
||||
"freezegun",
|
||||
"grpcio>=1.83.1",
|
||||
|
||||
+7
-1
@@ -19,7 +19,13 @@ testpaths =
|
||||
tests
|
||||
python_files = *_test.py test_*.py *_tests.py *viz/utils.py
|
||||
# `-p no:warnings` temporarily disabled in favor of more finely tuned `filterwarnings`.
|
||||
#addopts = -p no:warnings
|
||||
# `not testcontainers` excludes tests/testcontainers/ by default: those spin up
|
||||
# real Docker containers, and `testpaths = tests` would otherwise pull them into
|
||||
# every plain `pytest` run. The dedicated CI job (testcontainers.yml) overrides
|
||||
# this with an explicit `-m testcontainers` to run them.
|
||||
addopts = -m "not testcontainers"
|
||||
markers =
|
||||
testcontainers: exercises a real database via testcontainers-python (needs Docker); excluded by default, see .github/workflows/testcontainers.yml
|
||||
asyncio_mode = auto
|
||||
|
||||
# `ignore` is effectively equivalent to `-p no:warnings`.
|
||||
|
||||
@@ -16,5 +16,32 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
-e .[development,bigquery,cockroachdb,druid,duckdb,fastmcp,gevent,gsheets,mysql,postgres,presto,prophet,trino,thumbnails]
|
||||
-e .[development,bigquery,clickhouse,cockroachdb,crate,databend,druid,duckdb,elasticsearch,fastmcp,firebird,gevent,gsheets,monetdb,mongodb,mssql,mysql,oracle,postgres,presto,prophet,risingwave,starrocks,trino,thumbnails,ydb]
|
||||
-e ./superset-extensions-cli[test]
|
||||
# testcontainers-backed db_engine_specs tests (tests/testcontainers/) --
|
||||
# see .github/workflows/testcontainers.yml
|
||||
#
|
||||
# `db2` (the `ibm-db-sa`/`ibm-db` driver) and `oceanbase` (the `oceanbase_py`
|
||||
# driver) are both deliberately left out of the baseline dev install above:
|
||||
# `ibm-db` ships no Linux arm64 wheel, breaking the multi-platform
|
||||
# (amd64+arm64) dev Docker image build; `oceanbase_py` pins
|
||||
# `sqlalchemy-utils>=0.38.3,<0.39`, which conflicts outright with Superset's
|
||||
# own `sqlalchemy-utils==0.42.1` pin -- there's no version of both that can
|
||||
# coexist in one resolved environment. Both testcontainers CI jobs install
|
||||
# their driver on demand instead, only for their own matrix leg -- see
|
||||
# .github/workflows/testcontainers.yml.
|
||||
#
|
||||
# mariadb/timescaledb/yugabytedb need no testcontainers extra of their own:
|
||||
# they reuse the postgres/mysql container classes pointed at a different
|
||||
# image, and psycopg2-binary/mysqlclient are already pulled in above via
|
||||
# the postgres/mysql extras. Plain postgres/mysql obviously need nothing
|
||||
# extra either. clickhouse and starrocks also need no testcontainers extra:
|
||||
# ClickHouseContainer has no driver import of its own (clickhouse-connect,
|
||||
# pulled in above via the clickhouse extra, is all the test needs), and
|
||||
# StarRocks has no dedicated testcontainers module at all -- its test uses
|
||||
# a generic DockerContainer plus the same mysqlclient the mysql extra
|
||||
# already provides. databend/risingwave/firebird/ydb are the same story:
|
||||
# none has a dedicated testcontainers module, so each test uses a generic
|
||||
# DockerContainer plus whatever driver its own extra above already
|
||||
# provides.
|
||||
testcontainers[cockroachdb,cratedb,mongodb,mssql,mysql,oracle,postgres,trino]>=4.15.0,<5
|
||||
|
||||
@@ -12,10 +12,17 @@
|
||||
# apache-superset
|
||||
aiofile==3.9.0
|
||||
# via py-key-value-aio
|
||||
aiohappyeyeballs==2.7.1
|
||||
# via aiohttp
|
||||
aiohttp==3.14.3
|
||||
# via ydb
|
||||
aiosignal==1.4.0
|
||||
# via aiohttp
|
||||
alembic==1.15.2
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# flask-migrate
|
||||
# starrocks
|
||||
amqp==5.3.1
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -24,6 +31,8 @@ annotated-types==0.7.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# pydantic
|
||||
antlr4-python3-runtime==4.13.2
|
||||
# via pymongosql
|
||||
anyio==4.11.0
|
||||
# via
|
||||
# httpx
|
||||
@@ -42,9 +51,12 @@ apsw==3.50.1.0
|
||||
# shillelagh
|
||||
astroid==3.3.10
|
||||
# via pylint
|
||||
asyncmy2==0.2.21
|
||||
# via starrocks
|
||||
attrs==25.3.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# aiohttp
|
||||
# cattrs
|
||||
# cyclopts
|
||||
# jsonschema
|
||||
@@ -65,6 +77,7 @@ backports-tarfile==1.2.0
|
||||
backports-zstd==1.6.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# clickhouse-connect
|
||||
# flask-compress
|
||||
bcrypt==4.3.0
|
||||
# via
|
||||
@@ -117,8 +130,11 @@ celery==5.6.3
|
||||
certifi==2026.5.20
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# clickhouse-connect
|
||||
# elasticsearch
|
||||
# httpcore
|
||||
# httpx
|
||||
# opensearch-py
|
||||
# requests
|
||||
cffi==2.0.0
|
||||
# via
|
||||
@@ -160,6 +176,8 @@ click-repl==0.3.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# celery
|
||||
clickhouse-connect==1.7.2
|
||||
# via apache-superset
|
||||
cmdstanpy==1.1.0
|
||||
# via prophet
|
||||
colorama==0.4.6
|
||||
@@ -171,6 +189,8 @@ contourpy==1.0.7
|
||||
# via matplotlib
|
||||
coverage==7.6.8
|
||||
# via pytest-cov
|
||||
crate==2.2.1
|
||||
# via sqlalchemy-cratedb
|
||||
cron-descriptor==1.4.5
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -186,14 +206,20 @@ cryptography==50.0.1
|
||||
# authlib
|
||||
# google-auth
|
||||
# joserfc
|
||||
# oracledb
|
||||
# paramiko
|
||||
# pyjwt
|
||||
# pymysql
|
||||
# pyopenssl
|
||||
# secretstorage
|
||||
cycler==0.12.1
|
||||
# via matplotlib
|
||||
cyclopts==4.2.4
|
||||
# via fastmcp-slim
|
||||
databend-driver==0.34.2
|
||||
# via databend-sqlalchemy
|
||||
databend-sqlalchemy==0.5.5
|
||||
# via apache-superset
|
||||
db-dtypes==1.3.1
|
||||
# via pandas-gbq
|
||||
defusedxml==0.7.1
|
||||
@@ -216,8 +242,11 @@ dnspython==2.7.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# email-validator
|
||||
docker==7.0.0
|
||||
# via apache-superset
|
||||
# pymongo
|
||||
docker==7.2.0
|
||||
# via
|
||||
# apache-superset
|
||||
# testcontainers
|
||||
docstring-parser==0.17.0
|
||||
# via cyclopts
|
||||
docutils==0.22.2
|
||||
@@ -228,6 +257,10 @@ duckdb==1.5.5
|
||||
# duckdb-engine
|
||||
duckdb-engine==0.17.0
|
||||
# via apache-superset
|
||||
elasticsearch==7.17.13
|
||||
# via elasticsearch-dbapi
|
||||
elasticsearch-dbapi==0.2.13
|
||||
# via apache-superset
|
||||
email-validator==2.2.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -237,6 +270,8 @@ et-xmlfile==2.0.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# openpyxl
|
||||
events==0.5
|
||||
# via opensearch-py
|
||||
exceptiongroup==1.3.0
|
||||
# via fastmcp-slim
|
||||
fastmcp==3.4.7
|
||||
@@ -247,6 +282,10 @@ filelock==3.20.3
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# virtualenv
|
||||
firebird-base==2.0.3
|
||||
# via firebird-driver
|
||||
firebird-driver==2.0.3
|
||||
# via sqlalchemy-firebird
|
||||
flask==3.1.3
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -327,12 +366,18 @@ fonttools==4.60.2
|
||||
# via matplotlib
|
||||
freezegun==1.5.1
|
||||
# via apache-superset
|
||||
frozenlist==1.8.0
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
future==1.0.0
|
||||
# via pyhive
|
||||
geographiclib==2.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# geopy
|
||||
geojson==3.3.0
|
||||
# via sqlalchemy-cratedb
|
||||
geopy==2.4.1
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -389,6 +434,7 @@ grpcio==1.83.1
|
||||
# apache-superset
|
||||
# google-api-core
|
||||
# grpcio-status
|
||||
# ydb
|
||||
grpcio-status==1.60.1
|
||||
# via google-api-core
|
||||
gunicorn==26.2.0
|
||||
@@ -414,6 +460,7 @@ httpx==0.28.1
|
||||
# via
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
# testcontainers
|
||||
httpx-sse==0.4.1
|
||||
# via mcp
|
||||
humanize==4.12.3
|
||||
@@ -430,6 +477,7 @@ idna==3.15
|
||||
# httpx
|
||||
# requests
|
||||
# url-normalize
|
||||
# yarl
|
||||
importlib-metadata==8.7.0
|
||||
# via
|
||||
# keyring
|
||||
@@ -468,6 +516,7 @@ jmespath==1.1.0
|
||||
# via
|
||||
# boto3
|
||||
# botocore
|
||||
# pymongosql
|
||||
joserfc==1.7.2
|
||||
# via fastmcp-slim
|
||||
jsonpath-ng==1.8.0
|
||||
@@ -500,6 +549,8 @@ kombu==5.6.2
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# celery
|
||||
lark==1.3.1
|
||||
# via starrocks
|
||||
lazy-object-proxy==1.10.0
|
||||
# via openapi-spec-validator
|
||||
limits==5.1.0
|
||||
@@ -507,7 +558,9 @@ limits==5.1.0
|
||||
# -c requirements/base-constraint.txt
|
||||
# flask-limiter
|
||||
lz4==4.4.5
|
||||
# via trino
|
||||
# via
|
||||
# clickhouse-connect
|
||||
# trino
|
||||
mako==1.4.1
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -568,6 +621,10 @@ msgspec==0.19.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# flask-session
|
||||
multidict==6.7.1
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
mysqlclient==2.2.8
|
||||
# via apache-superset
|
||||
nh3==0.3.7
|
||||
@@ -606,14 +663,22 @@ openpyxl==3.1.5
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# pandas
|
||||
opensearch-py==2.8.0
|
||||
# via elasticsearch-dbapi
|
||||
opentelemetry-api==1.39.1
|
||||
# via fastmcp-slim
|
||||
oracledb==4.0.2
|
||||
# via
|
||||
# apache-superset
|
||||
# testcontainers
|
||||
ordered-set==4.1.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# flask-limiter
|
||||
orjson==3.11.9
|
||||
# via trino
|
||||
# via
|
||||
# crate
|
||||
# trino
|
||||
packaging==25.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -621,8 +686,8 @@ packaging==25.0
|
||||
# apispec
|
||||
# db-dtypes
|
||||
# deprecation
|
||||
# docker
|
||||
# duckdb-engine
|
||||
# elasticsearch-dbapi
|
||||
# fastmcp-slim
|
||||
# google-cloud-bigquery
|
||||
# kombu
|
||||
@@ -632,6 +697,8 @@ packaging==25.0
|
||||
# pytest
|
||||
# shillelagh
|
||||
# sqlalchemy-bigquery
|
||||
# sqlalchemy-firebird
|
||||
# ydb
|
||||
pandas==2.3.3
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -693,16 +760,22 @@ prompt-toolkit==3.0.51
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# click-repl
|
||||
propcache==0.5.2
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
prophet==1.4.0
|
||||
# via apache-superset
|
||||
proto-plus==1.25.0
|
||||
# via google-api-core
|
||||
protobuf==5.29.6
|
||||
# via
|
||||
# firebird-base
|
||||
# google-api-core
|
||||
# googleapis-common-protos
|
||||
# grpcio-status
|
||||
# proto-plus
|
||||
# ydb
|
||||
psutil==6.1.0
|
||||
# via
|
||||
# apache-superset
|
||||
@@ -776,6 +849,24 @@ pyjwt==2.13.0
|
||||
# mcp
|
||||
pylint==3.3.7
|
||||
# via apache-superset
|
||||
pymonetdb==1.9.1
|
||||
# via
|
||||
# apache-superset
|
||||
# sqlalchemy-monetdb
|
||||
pymongo==4.17.0
|
||||
# via
|
||||
# pymongosql
|
||||
# testcontainers
|
||||
pymongosql==0.7.3
|
||||
# via apache-superset
|
||||
pymssql==2.3.13
|
||||
# via
|
||||
# apache-superset
|
||||
# testcontainers
|
||||
pymysql==1.2.0
|
||||
# via
|
||||
# starrocks
|
||||
# testcontainers
|
||||
pynacl==1.6.2
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -820,11 +911,13 @@ python-dateutil==2.9.0.post0
|
||||
# botocore
|
||||
# celery
|
||||
# croniter
|
||||
# firebird-driver
|
||||
# flask-appbuilder
|
||||
# freezegun
|
||||
# google-cloud-bigquery
|
||||
# holidays
|
||||
# matplotlib
|
||||
# opensearch-py
|
||||
# pandas
|
||||
# pyhive
|
||||
# shillelagh
|
||||
@@ -835,6 +928,7 @@ python-dotenv==1.2.2
|
||||
# apache-superset
|
||||
# fastmcp-slim
|
||||
# pydantic-settings
|
||||
# testcontainers
|
||||
python-ldap==3.4.7
|
||||
# via apache-superset
|
||||
python-multipart==0.0.29
|
||||
@@ -878,6 +972,7 @@ requests==2.33.0
|
||||
# google-api-core
|
||||
# google-cloud-bigquery
|
||||
# jsonschema-path
|
||||
# opensearch-py
|
||||
# pydruid
|
||||
# pyhive
|
||||
# requests-cache
|
||||
@@ -956,7 +1051,9 @@ sqlalchemy==2.0.52
|
||||
# alembic
|
||||
# apache-superset
|
||||
# apache-superset-core
|
||||
# databend-sqlalchemy
|
||||
# duckdb-engine
|
||||
# elasticsearch-dbapi
|
||||
# flask-appbuilder
|
||||
# flask-sqlalchemy
|
||||
# marshmallow-sqlalchemy
|
||||
@@ -964,7 +1061,14 @@ sqlalchemy==2.0.52
|
||||
# sqlalchemy-bigquery
|
||||
# sqlalchemy-cockroachdb
|
||||
# sqlalchemy-continuum
|
||||
# sqlalchemy-cratedb
|
||||
# sqlalchemy-firebird
|
||||
# sqlalchemy-monetdb
|
||||
# sqlalchemy-risingwave
|
||||
# sqlalchemy-utils
|
||||
# starrocks
|
||||
# testcontainers
|
||||
# ydb-sqlalchemy
|
||||
sqlalchemy-bigquery==1.17.2
|
||||
# via apache-superset
|
||||
sqlalchemy-cockroachdb==2.0.4
|
||||
@@ -973,6 +1077,16 @@ sqlalchemy-continuum==1.7.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
sqlalchemy-cratedb==0.43.1
|
||||
# via
|
||||
# apache-superset
|
||||
# testcontainers
|
||||
sqlalchemy-firebird==2.2.0
|
||||
# via apache-superset
|
||||
sqlalchemy-monetdb==2.1.0
|
||||
# via apache-superset
|
||||
sqlalchemy-risingwave==2.1.0
|
||||
# via apache-superset
|
||||
sqlalchemy-utils==0.42.1
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -984,6 +1098,7 @@ sqlglot==30.17.0
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
# apache-superset-core
|
||||
# ydb-sqlglot-plugin
|
||||
sqloxide==0.1.51
|
||||
# via apache-superset
|
||||
sse-starlette==3.0.2
|
||||
@@ -996,6 +1111,8 @@ starlette==1.3.1
|
||||
# via
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
starrocks==1.3.4
|
||||
# via apache-superset
|
||||
statsd==4.0.1
|
||||
# via apache-superset
|
||||
syntaqlite==0.9.0
|
||||
@@ -1004,6 +1121,8 @@ tabulate==0.10.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
testcontainers==4.15.0
|
||||
# via -r requirements/development.in
|
||||
tiktoken==0.14.0
|
||||
# via apache-superset
|
||||
tomli-w==1.2.0
|
||||
@@ -1015,10 +1134,14 @@ tqdm==4.67.1
|
||||
# cmdstanpy
|
||||
# prophet
|
||||
trino==0.339.0
|
||||
# via apache-superset
|
||||
# via
|
||||
# apache-superset
|
||||
# testcontainers
|
||||
typing-extensions==4.16.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
# alembic
|
||||
# anyio
|
||||
# apache-superset
|
||||
@@ -1030,6 +1153,7 @@ typing-extensions==4.16.0
|
||||
# limits
|
||||
# mcp
|
||||
# opentelemetry-api
|
||||
# oracledb
|
||||
# py-key-value-aio
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
@@ -1038,6 +1162,7 @@ typing-extensions==4.16.0
|
||||
# shillelagh
|
||||
# sqlalchemy
|
||||
# starlette
|
||||
# testcontainers
|
||||
# typing-inspection
|
||||
typing-inspection==0.4.2
|
||||
# via
|
||||
@@ -1065,13 +1190,22 @@ urllib3==2.7.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# botocore
|
||||
# clickhouse-connect
|
||||
# crate
|
||||
# docker
|
||||
# elasticsearch
|
||||
# opensearch-py
|
||||
# requests
|
||||
# requests-cache
|
||||
# testcontainers
|
||||
uvicorn==0.37.0
|
||||
# via
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
verlib2==0.3.2
|
||||
# via
|
||||
# crate
|
||||
# sqlalchemy-cratedb
|
||||
vine==5.1.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -1105,6 +1239,7 @@ wrapt==1.17.2
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# deprecated
|
||||
# testcontainers
|
||||
wtforms==3.2.2
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
@@ -1125,6 +1260,18 @@ xlsxwriter==3.2.9
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
# pandas
|
||||
yarl==1.24.5
|
||||
# via aiohttp
|
||||
ydb==3.31.4
|
||||
# via
|
||||
# ydb-dbapi
|
||||
# ydb-sqlalchemy
|
||||
ydb-dbapi==0.1.23
|
||||
# via ydb-sqlalchemy
|
||||
ydb-sqlalchemy==0.1.22
|
||||
# via apache-superset
|
||||
ydb-sqlglot-plugin==0.2.8
|
||||
# via apache-superset
|
||||
zipp==3.23.0
|
||||
# via importlib-metadata
|
||||
zope-event==5.0
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
AxisType,
|
||||
buildCustomFormatters,
|
||||
CategoricalColorNamespace,
|
||||
ComparisonType,
|
||||
CurrencyFormatter,
|
||||
DataRecordValue,
|
||||
DTTM_ALIAS,
|
||||
@@ -419,6 +420,11 @@ export default function transformProps(
|
||||
|
||||
const refs: Refs = {};
|
||||
const groupBy = ensureIsArray(groupby);
|
||||
// Series whose `label_map` entry led with a time offset, recorded before the shift
|
||||
// below drops it. That leading column is the only structural marker distinguishing a
|
||||
// derived comparison row from a base row whose dimension value happens to read like
|
||||
// the offset, and it is gone from `labelMap` by the time the formatters run.
|
||||
const derivedComparisonSeries = new Set<string>();
|
||||
const labelMap: { [key: string]: string[] } = Object.entries(
|
||||
label_map,
|
||||
).reduce((acc, entry) => {
|
||||
@@ -427,6 +433,7 @@ export default function transformProps(
|
||||
Array.isArray(timeCompare) &&
|
||||
timeCompare.includes(entry[1][0])
|
||||
) {
|
||||
derivedComparisonSeries.add(entry[0]);
|
||||
entry[1].shift();
|
||||
}
|
||||
return { ...acc, [entry[0]]: entry[1] };
|
||||
@@ -681,6 +688,51 @@ export default function transformProps(
|
||||
const array = ensureIsArray(chartProps.rawFormData?.time_compare);
|
||||
const inverted = invert(verboseMap);
|
||||
|
||||
// A Percentage or Ratio time comparison replaces the derived series' values with a
|
||||
// dimensionless number, so that row is no longer in the source metric's units and
|
||||
// must not inherit its currency/D3 format.
|
||||
//
|
||||
// `label_map` carries the structured identity behind a rendered series name, and
|
||||
// `renameOperator` puts the offset at the front of a derived row's entry:
|
||||
//
|
||||
// derived '1 week ago, East' -> ['1 week ago', 'East']
|
||||
// derived 'count, 1 year ago' -> ['1 year ago', 'count']
|
||||
// base 'sum__num, East' -> ['sum__num', 'East']
|
||||
//
|
||||
// so the leading column says which it is. Matching the rendered name instead would
|
||||
// misread a base series whose dimension value happens to equal the offset — a region
|
||||
// literally named "1 week ago" gives 'sum__num, 1 week ago', which reads as derived.
|
||||
const isDerivedComparisonSeries = (seriesKey: string) => {
|
||||
// Recorded above, before the offset was shifted off the `label_map` entry.
|
||||
if (derivedComparisonSeries.has(seriesKey)) {
|
||||
return true;
|
||||
}
|
||||
const columns = labelMap?.[seriesKey];
|
||||
// The shift only runs when `timeCompare` is populated; otherwise the entry still
|
||||
// leads with the offset and can be read directly.
|
||||
return columns?.length
|
||||
? array.includes(columns[0])
|
||||
: array.includes(seriesKey);
|
||||
};
|
||||
|
||||
// Percentage yields `(s - c) / c`, which reads as a percentage. Ratio yields `s / c`,
|
||||
// a plain multiplier, so it takes a unitless number format rather than a percent one.
|
||||
const ratioFormatter = getNumberFormatter(NumberFormats.SMART_NUMBER);
|
||||
|
||||
const getComparisonFormatter = (seriesKey: string) => {
|
||||
if (!isDerivedComparisonSeries(seriesKey)) {
|
||||
return undefined;
|
||||
}
|
||||
switch (chartProps.rawFormData?.comparison_type) {
|
||||
case ComparisonType.Percentage:
|
||||
return percentFormatter;
|
||||
case ComparisonType.Ratio:
|
||||
return ratioFormatter;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
// With the "full range" time-shift option, offset series are outer-joined onto
|
||||
// the main series, which inserts null rows into the main series wherever the
|
||||
// comparison period has data the current period lacks. Connect nulls so the
|
||||
@@ -1522,6 +1574,31 @@ export default function transformProps(
|
||||
value.forecastTrend || value.forecastLower || value.forecastUpper,
|
||||
);
|
||||
|
||||
// Resolve the value formatter per series so each metric keeps its own
|
||||
// D3/currency format, matching how the series labels are formatted.
|
||||
// Without the series key, `getCustomFormatter` returns undefined for
|
||||
// multi-metric charts and every row falls back to `defaultFormatter`,
|
||||
// rendering the y-axis/currency format for all metrics.
|
||||
//
|
||||
// The tooltip key is the rendered series name, so resolve it through
|
||||
// `labelMap`, whose values lead with the raw metric label. Series
|
||||
// renamed by a verbose_name are absent from that map, so fall back to
|
||||
// the verbose-name inversion, as MixedTimeseries does. A Percentage or
|
||||
// Ratio comparison row is dimensionless rather than a value in the
|
||||
// metric's units, so it takes its own formatter instead of the metric's.
|
||||
const getSeriesFormatter = (seriesKey: string) =>
|
||||
forcePercentFormatter
|
||||
? percentFormatter
|
||||
: (getComparisonFormatter(seriesKey) ??
|
||||
getCustomFormatter(
|
||||
customFormatters,
|
||||
metrics,
|
||||
labelMap?.[seriesKey]?.[0] ?? inverted[seriesKey],
|
||||
) ??
|
||||
defaultFormatter);
|
||||
|
||||
// The total row aggregates every series, so it keeps the chart-level
|
||||
// formatter rather than any single metric's format.
|
||||
const formatter = forcePercentFormatter
|
||||
? percentFormatter
|
||||
: (getCustomFormatter(customFormatters, metrics) ?? defaultFormatter);
|
||||
@@ -1552,7 +1629,7 @@ export default function transformProps(
|
||||
const row = formatForecastTooltipSeries({
|
||||
...value,
|
||||
seriesName: key,
|
||||
formatter,
|
||||
formatter: getSeriesFormatter(key),
|
||||
marker,
|
||||
truncation: tooltipTruncation,
|
||||
});
|
||||
|
||||
+579
@@ -3576,3 +3576,582 @@ test('boundary label alignment is dropped when the orientation moves the time ax
|
||||
expect(horizontal.axisLabel.showMinLabel).toBe(true);
|
||||
expect(horizontal.axisLabel.showMaxLabel).toBe(true);
|
||||
});
|
||||
|
||||
test('tooltip formats each series with its own metric format instead of the default formatter', () => {
|
||||
// Two saved metrics with different formats: `pct_change` carries a percentage
|
||||
// D3 format, `count` carries a currency format. The series labels already
|
||||
// honor each metric's format; the tooltip must do the same.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metrics: ['count', 'pct_change'],
|
||||
richTooltip: true,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[{ count: 1000, pct_change: 0.1234, __timestamp: BASE_TIMESTAMP }],
|
||||
{ label_map: { count: ['count'], pct_change: ['pct_change'] } },
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: { pct_change: '.2%' },
|
||||
currencyFormats: { count: { symbol: 'USD', symbolPosition: 'prefix' } },
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{ seriesId: 'count', seriesName: 'count', value: [BASE_TIMESTAMP, 1000] },
|
||||
{
|
||||
seriesId: 'pct_change',
|
||||
seriesName: 'pct_change',
|
||||
value: [BASE_TIMESTAMP, 0.1234],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('12.34%');
|
||||
expect(result).toContain('$');
|
||||
});
|
||||
|
||||
test('tooltip resolves per-metric formats for series renamed by verbose_name', () => {
|
||||
// With a verbose_name configured, the rendered series name (and so the
|
||||
// tooltip key) is the verbose label, while `label_map` stays keyed by the
|
||||
// raw metric label. The formatter lookup has to bridge that gap.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metrics: ['count', 'pct_change'],
|
||||
richTooltip: true,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[{ count: 1000, pct_change: 0.1234, __timestamp: BASE_TIMESTAMP }],
|
||||
{ label_map: { count: ['count'], pct_change: ['pct_change'] } },
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: { count: 'Total Count', pct_change: 'Percent Change' },
|
||||
columnFormats: { pct_change: '.2%' },
|
||||
currencyFormats: { count: { symbol: 'USD', symbolPosition: 'prefix' } },
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'Total Count',
|
||||
seriesName: 'Total Count',
|
||||
value: [BASE_TIMESTAMP, 1000],
|
||||
},
|
||||
{
|
||||
seriesId: 'Percent Change',
|
||||
seriesName: 'Percent Change',
|
||||
value: [BASE_TIMESTAMP, 0.1234],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('12.34%');
|
||||
expect(result).toContain('$');
|
||||
});
|
||||
|
||||
test('tooltip keeps per-metric formats on time-comparison (time-shifted) series', () => {
|
||||
// A time-shifted series renders under a name carrying the offset, and its
|
||||
// `label_map` entry leads with that offset rather than the metric. The
|
||||
// formatter lookup has to land on the underlying metric so the shifted row is
|
||||
// formatted like the series it is compared against.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metrics: ['count', 'pct_change'],
|
||||
richTooltip: true,
|
||||
timeCompare: ['1 year ago'],
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
count: 1000,
|
||||
pct_change: 0.1234,
|
||||
'count, 1 year ago': 900,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
count: ['count'],
|
||||
pct_change: ['pct_change'],
|
||||
'count, 1 year ago': ['1 year ago', 'count'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: { pct_change: '.2%' },
|
||||
currencyFormats: { count: { symbol: 'USD', symbolPosition: 'prefix' } },
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{ seriesId: 'count', seriesName: 'count', value: [BASE_TIMESTAMP, 1000] },
|
||||
{
|
||||
seriesId: 'count, 1 year ago',
|
||||
seriesName: 'count, 1 year ago',
|
||||
value: [BASE_TIMESTAMP, 900],
|
||||
},
|
||||
]);
|
||||
|
||||
// The base series and its time-shifted counterpart keep the currency format.
|
||||
expect(result).toContain('$ 1k');
|
||||
expect(result).toContain('$ 900');
|
||||
});
|
||||
|
||||
test('tooltip does not apply a metric currency format to a Percentage time comparison', () => {
|
||||
// Reported on #33757: a Time Comparison set to Percentage change on a
|
||||
// currency metric kept rendering the derived row in dollars. That row holds a
|
||||
// ratio rather than a value in the metric's units, so it must not inherit the
|
||||
// metric's saved CurrencyFormatter.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Percentage,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[{ sum__num: 100, '1 week ago': 0.25, __timestamp: BASE_TIMESTAMP }],
|
||||
{ label_map: { sum__num: ['sum__num'], '1 week ago': ['1 week ago'] } },
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num',
|
||||
seriesName: 'sum__num',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago',
|
||||
seriesName: '1 week ago',
|
||||
value: [BASE_TIMESTAMP, 0.25],
|
||||
},
|
||||
]);
|
||||
|
||||
// The source metric keeps its currency; the percentage-change row does not.
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('25.00%');
|
||||
expect(result).not.toContain('$ 0.25');
|
||||
});
|
||||
|
||||
test('tooltip does not apply a metric currency format to a grouped Percentage time comparison', () => {
|
||||
// A groupby appends the dimension values to the derived series name
|
||||
// ("1 week ago, East"), so matching the dimensionless names alone left the
|
||||
// grouped rows resolving back to the source metric's CurrencyFormatter.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
groupby: ['region'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Percentage,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
'sum__num, East': 100,
|
||||
'1 week ago, East': 0.25,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
'sum__num, East': ['sum__num', 'East'],
|
||||
'1 week ago, East': ['1 week ago', 'East'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num, East',
|
||||
seriesName: 'sum__num, East',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago, East',
|
||||
seriesName: '1 week ago, East',
|
||||
value: [BASE_TIMESTAMP, 0.25],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('25.00%');
|
||||
expect(result).not.toContain('$ 0.25');
|
||||
});
|
||||
|
||||
test('tooltip does not apply a metric currency format to a Ratio time comparison', () => {
|
||||
// A Ratio comparison is `source / compare`, a plain multiplier, so the derived row is
|
||||
// no more in the metric's currency than a Percentage one is — but it is not a
|
||||
// percentage either, so it takes a unitless number format rather than the percent one.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Ratio,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[{ sum__num: 100, '1 week ago': 1.25, __timestamp: BASE_TIMESTAMP }],
|
||||
{ label_map: { sum__num: ['sum__num'], '1 week ago': ['1 week ago'] } },
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num',
|
||||
seriesName: 'sum__num',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago',
|
||||
seriesName: '1 week ago',
|
||||
value: [BASE_TIMESTAMP, 1.25],
|
||||
},
|
||||
]);
|
||||
|
||||
// The source metric keeps its currency; the ratio row renders as a plain number.
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('1.25');
|
||||
expect(result).not.toContain('$ 1.25');
|
||||
});
|
||||
|
||||
test('tooltip does not apply a metric currency format to a grouped Ratio time comparison', () => {
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
groupby: ['region'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Ratio,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
'sum__num, East': 100,
|
||||
'1 week ago, East': 1.25,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
'sum__num, East': ['sum__num', 'East'],
|
||||
'1 week ago, East': ['1 week ago', 'East'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num, East',
|
||||
seriesName: 'sum__num, East',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago, East',
|
||||
seriesName: '1 week ago, East',
|
||||
value: [BASE_TIMESTAMP, 1.25],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('1.25');
|
||||
expect(result).not.toContain('$ 1.25');
|
||||
});
|
||||
|
||||
test('tooltip formats derived rows when timeCompare normalization strips the offset', () => {
|
||||
// With `timeCompare` populated, `labelMap` has its leading offset shifted off before
|
||||
// the formatters run, so the derived identity has to be captured during that pass —
|
||||
// reading `labelMap[key][0]` afterwards sees the dimension value instead.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
groupby: ['region'],
|
||||
richTooltip: true,
|
||||
timeCompare: ['1 week ago'],
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Percentage,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
'sum__num, East': 100,
|
||||
'1 week ago, East': 0.25,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
'sum__num, East': ['sum__num', 'East'],
|
||||
'1 week ago, East': ['1 week ago', 'East'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num, East',
|
||||
seriesName: 'sum__num, East',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago, East',
|
||||
seriesName: '1 week ago, East',
|
||||
value: [BASE_TIMESTAMP, 0.25],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('25.00%');
|
||||
expect(result).not.toContain('$ 0.25');
|
||||
});
|
||||
|
||||
test('tooltip gives a Ratio row a unitless format when timeCompare is set', () => {
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
groupby: ['region'],
|
||||
richTooltip: true,
|
||||
timeCompare: ['1 week ago'],
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Ratio,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
'sum__num, East': 100,
|
||||
'1 week ago, East': 1.25,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
'sum__num, East': ['sum__num', 'East'],
|
||||
'1 week ago, East': ['1 week ago', 'East'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num, East',
|
||||
seriesName: 'sum__num, East',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago, East',
|
||||
seriesName: '1 week ago, East',
|
||||
value: [BASE_TIMESTAMP, 1.25],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('1.25');
|
||||
expect(result).not.toContain('$ 1.25');
|
||||
});
|
||||
|
||||
test('tooltip keeps the metric format when a dimension value equals the offset', () => {
|
||||
// A groupby value can legitimately read like the configured offset, giving a *base*
|
||||
// series called `sum__num, 1 week ago`. Matching the rendered name would classify it
|
||||
// as derived and strip its currency; `label_map` leads with the metric, not the
|
||||
// offset, so it stays a base row.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
groupby: ['region'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Percentage,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[
|
||||
{
|
||||
'sum__num, 1 week ago': 100,
|
||||
'1 week ago, 1 week ago': 0.25,
|
||||
__timestamp: BASE_TIMESTAMP,
|
||||
},
|
||||
],
|
||||
{
|
||||
label_map: {
|
||||
// The region is named "1 week ago"; the metric still leads the base entry.
|
||||
'sum__num, 1 week ago': ['sum__num', '1 week ago'],
|
||||
// Its derived counterpart leads with the offset.
|
||||
'1 week ago, 1 week ago': ['1 week ago', '1 week ago'],
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num, 1 week ago',
|
||||
seriesName: 'sum__num, 1 week ago',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago, 1 week ago',
|
||||
seriesName: '1 week ago, 1 week ago',
|
||||
value: [BASE_TIMESTAMP, 0.25],
|
||||
},
|
||||
]);
|
||||
|
||||
// The base row keeps its currency even though its name ends in the offset, and the
|
||||
// genuinely derived row is still formatted as a percentage.
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('25.00%');
|
||||
});
|
||||
|
||||
test('tooltip keeps the metric format on a Difference time comparison', () => {
|
||||
// Difference is `source - compare`, which stays in the metric's units, so unlike
|
||||
// Percentage and Ratio it must keep the currency format.
|
||||
const chartProps = createTestChartProps({
|
||||
formData: {
|
||||
metric: 'sum__num',
|
||||
metrics: ['sum__num'],
|
||||
richTooltip: true,
|
||||
time_compare: ['1 week ago'],
|
||||
comparison_type: ComparisonType.Difference,
|
||||
},
|
||||
queriesData: [
|
||||
createTestQueryData(
|
||||
[{ sum__num: 100, '1 week ago': 25, __timestamp: BASE_TIMESTAMP }],
|
||||
{ label_map: { sum__num: ['sum__num'], '1 week ago': ['1 week ago'] } },
|
||||
),
|
||||
],
|
||||
datasource: {
|
||||
verboseMap: {},
|
||||
columnFormats: {},
|
||||
currencyFormats: {
|
||||
sum__num: { symbol: 'USD', symbolPosition: 'prefix' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const { echartOptions } = transformProps(chartProps);
|
||||
const { tooltip } = echartOptions as unknown as TooltipFormatterOptions;
|
||||
|
||||
const result = tooltip.formatter([
|
||||
{
|
||||
seriesId: 'sum__num',
|
||||
seriesName: 'sum__num',
|
||||
value: [BASE_TIMESTAMP, 100],
|
||||
},
|
||||
{
|
||||
seriesId: '1 week ago',
|
||||
seriesName: '1 week ago',
|
||||
value: [BASE_TIMESTAMP, 25],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toContain('$ 100');
|
||||
expect(result).toContain('$ 25');
|
||||
});
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/** @jsxImportSource @emotion/react */
|
||||
import {
|
||||
Children,
|
||||
cloneElement,
|
||||
@@ -286,9 +288,28 @@ function StickyWrap({
|
||||
</colgroup>
|
||||
);
|
||||
|
||||
const headerContainerWidth = hasVerticalScroll
|
||||
? maxWidth - scrollBarSize
|
||||
: maxWidth;
|
||||
// Below, `width: maxWidth` is applied unconditionally (never reduced by
|
||||
// subtracting a separately-measured scrollbar width, unlike this file's
|
||||
// previous `maxWidth - scrollBarSize`). That's the load-bearing part of
|
||||
// this fix: the shared colgroup (computed from the sizer below, whose
|
||||
// own clientWidth can only ever be <= maxWidth) can never need more
|
||||
// width than that, so a header/footer wrapper that's never narrowed
|
||||
// below maxWidth can never clip it, regardless of whether any
|
||||
// JS-measured scrollbar size agrees with what the sizer/body actually
|
||||
// reserve in a given browser.
|
||||
//
|
||||
// `scrollbarGutter`/`scrollBarStyles` below are a separate, secondary
|
||||
// measure -- matching an actual clip boundary is not what they're for
|
||||
// (an `overflow: hidden` box's clip boundary sits at its real
|
||||
// border-box edge regardless of `scrollbar-gutter`, which only affects
|
||||
// what `clientWidth` reports). They keep header/footer's reported
|
||||
// `clientWidth` consistent with body's so that, when both a vertical
|
||||
// and a horizontal scrollbar are present, the horizontal `scrollLeft`
|
||||
// synced from body (see `onScroll` below) reveals the same slice of the
|
||||
// row in header/footer as is actually visible in body.
|
||||
const headerFooterGutter: CSSProperties = {
|
||||
scrollbarGutter: hasVerticalScroll ? 'stable' : undefined,
|
||||
};
|
||||
|
||||
headerTable = (
|
||||
<div
|
||||
@@ -296,9 +317,11 @@ function StickyWrap({
|
||||
ref={scrollHeaderRef}
|
||||
style={{
|
||||
overflow: 'hidden',
|
||||
width: headerContainerWidth,
|
||||
width: maxWidth,
|
||||
boxSizing: 'border-box',
|
||||
...headerFooterGutter,
|
||||
}}
|
||||
css={scrollBarStyles}
|
||||
role="presentation"
|
||||
>
|
||||
{cloneElement(
|
||||
@@ -317,9 +340,11 @@ function StickyWrap({
|
||||
ref={scrollFooterRef}
|
||||
style={{
|
||||
overflow: 'hidden',
|
||||
width: headerContainerWidth,
|
||||
width: maxWidth,
|
||||
boxSizing: 'border-box',
|
||||
...headerFooterGutter,
|
||||
}}
|
||||
css={scrollBarStyles}
|
||||
role="presentation"
|
||||
>
|
||||
{cloneElement(
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { useCallback } from 'react';
|
||||
import { useTable, Column } from 'react-table';
|
||||
import { render } from '@superset-ui/core/spec';
|
||||
import useSticky from '../../../src/DataTable/hooks/useSticky';
|
||||
|
||||
// A value distinguishable from any real scrollbar width, so the width
|
||||
// assertions below can detect whether header/footer's wrapper width was
|
||||
// computed by subtracting this JS-measured probe from `maxWidth` (the old,
|
||||
// removed `maxWidth - scrollBarSize` behavior) rather than always being the
|
||||
// unconditional `maxWidth` the fix uses. If that subtraction is ever
|
||||
// reintroduced, header/footer's `style.width` would read
|
||||
// `${MAX_WIDTH - MOCKED_SCROLLBAR_PROBE_SIZE}px`, an unmistakably wrong
|
||||
// value given how large this mock is.
|
||||
const MOCKED_SCROLLBAR_PROBE_SIZE = 42;
|
||||
|
||||
jest.mock('../../../src/DataTable/utils/getScrollBarSize', () => ({
|
||||
__esModule: true,
|
||||
CUSTOM_SCROLLBAR_SIZE: 8,
|
||||
default: () => 0,
|
||||
getCustomScrollBarSize: () => MOCKED_SCROLLBAR_PROBE_SIZE,
|
||||
}));
|
||||
|
||||
const MAX_WIDTH = 300;
|
||||
const MAX_HEIGHT = 120; // small enough that the mocked content forces a vertical scroll
|
||||
|
||||
const TOTAL_HEADER_HEIGHT = 30;
|
||||
const TOTAL_FOOTER_HEIGHT = 30;
|
||||
// Larger than `MAX_HEIGHT - TOTAL_HEADER_HEIGHT - TOTAL_FOOTER_HEIGHT`, so the
|
||||
// sticky layout effect computes `hasVerticalScroll: true`.
|
||||
const FULL_TABLE_HEIGHT = 400;
|
||||
|
||||
function mockMeasurements() {
|
||||
jest
|
||||
.spyOn(HTMLElement.prototype, 'clientHeight', 'get')
|
||||
.mockImplementation(function mockClientHeight(this: HTMLElement) {
|
||||
if (this.tagName === 'THEAD') return TOTAL_HEADER_HEIGHT;
|
||||
if (this.tagName === 'TFOOT') return TOTAL_FOOTER_HEIGHT;
|
||||
if (this.tagName === 'TABLE') return FULL_TABLE_HEIGHT;
|
||||
return 0;
|
||||
});
|
||||
jest
|
||||
.spyOn(HTMLElement.prototype, 'getBoundingClientRect')
|
||||
.mockImplementation(function mockRect(this: HTMLElement) {
|
||||
const width = this.tagName === 'TH' ? 60 : 0;
|
||||
return {
|
||||
width,
|
||||
height: 0,
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: width,
|
||||
bottom: 0,
|
||||
x: 0,
|
||||
y: 0,
|
||||
toJSON: () => {},
|
||||
} as DOMRect;
|
||||
});
|
||||
}
|
||||
|
||||
type Row = { category: string; amount: string };
|
||||
|
||||
const columns: Column<Row>[] = [
|
||||
{ Header: 'Category', accessor: 'category' },
|
||||
{ Header: 'SUM(amount)', accessor: 'amount' },
|
||||
];
|
||||
|
||||
const data: Row[] = Array.from({ length: 8 }, (_, i) => ({
|
||||
category: `Category ${i}`,
|
||||
amount: `${1234567.891234 + i}`,
|
||||
}));
|
||||
|
||||
function StickyTableHarness() {
|
||||
const getTableSize = useCallback(
|
||||
() => ({ width: MAX_WIDTH, height: MAX_HEIGHT }),
|
||||
[],
|
||||
);
|
||||
const { getTableProps, headerGroups, rows, prepareRow, wrapStickyTable } =
|
||||
useTable<Row>(
|
||||
{
|
||||
columns,
|
||||
data,
|
||||
getTableSize,
|
||||
},
|
||||
useSticky,
|
||||
);
|
||||
|
||||
const renderTable = () => (
|
||||
<table {...getTableProps()}>
|
||||
<thead>
|
||||
{headerGroups.map(hg => (
|
||||
<tr {...hg.getHeaderGroupProps()} key={hg.id}>
|
||||
{hg.headers.map(col => (
|
||||
<th {...col.getHeaderProps()} key={col.id}>
|
||||
{col.render('Header')}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map(row => {
|
||||
prepareRow(row);
|
||||
return (
|
||||
<tr {...row.getRowProps()} key={row.id}>
|
||||
{row.cells.map(cell => (
|
||||
<td {...cell.getCellProps()} key={cell.column.id}>
|
||||
{cell.render('Cell')}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
<tfoot>
|
||||
<tr key="footer">
|
||||
<th>Summary</th>
|
||||
<td>
|
||||
<strong>14814904.694808</strong>
|
||||
</td>
|
||||
</tr>
|
||||
</tfoot>
|
||||
</table>
|
||||
);
|
||||
|
||||
return <div data-test="sticky-root">{wrapStickyTable(renderTable)}</div>;
|
||||
}
|
||||
|
||||
test('sticky header/footer width matches the body, independent of the scrollbar-size probe', () => {
|
||||
mockMeasurements();
|
||||
|
||||
const { container } = render(<StickyTableHarness />);
|
||||
|
||||
const root = container.querySelector('[data-test="sticky-root"] > div');
|
||||
expect(root).not.toBeNull();
|
||||
const [headerDiv, bodyDiv, footerDiv] = Array.from(
|
||||
root!.children,
|
||||
) as HTMLDivElement[];
|
||||
|
||||
expect(bodyDiv.style.width).toBe(`${MAX_WIDTH}px`);
|
||||
|
||||
// This is the load-bearing assertion for the reported bug. Before the fix
|
||||
// these read `${MAX_WIDTH - MOCKED_SCROLLBAR_PROBE_SIZE}px` (258px) --
|
||||
// genuinely narrower than the body, from a real CSS `width` subtraction
|
||||
// (`maxWidth - scrollBarSize`), not just a smaller reported `clientWidth`.
|
||||
// A wrapper that's actually narrower than the shared, fixed-layout
|
||||
// colgroup it has to display gets genuinely clipped by its own
|
||||
// `overflow: hidden` (verified with real hit-testing in a real browser --
|
||||
// this is not true of the `scrollbarGutter` assertions below). The fix
|
||||
// makes header/footer always exactly `maxWidth`, which the colgroup
|
||||
// (bounded by the sizer's `clientWidth`, itself bounded by `maxWidth`)
|
||||
// can never exceed.
|
||||
expect(headerDiv.style.width).toBe(`${MAX_WIDTH}px`);
|
||||
expect(footerDiv.style.width).toBe(`${MAX_WIDTH}px`);
|
||||
|
||||
// Secondary, not itself load-bearing for preventing clipping: real
|
||||
// hit-testing shows `scrollbar-gutter` on an `overflow: hidden` box
|
||||
// changes what `clientWidth` reports without moving where it actually
|
||||
// clips, so this doesn't guard against the reported bug by itself. It's
|
||||
// asserted because header/footer's reported `clientWidth` still needs to
|
||||
// match body's `clientWidth` for their programmatically
|
||||
// synced `scrollLeft` (see `onScroll` in `useSticky.tsx`) to reveal the
|
||||
// same slice of the row body actually shows, when a horizontal scrollbar
|
||||
// is present alongside a vertical one.
|
||||
expect(headerDiv.style.scrollbarGutter).toBe(bodyDiv.style.scrollbarGutter);
|
||||
expect(footerDiv.style.scrollbarGutter).toBe(bodyDiv.style.scrollbarGutter);
|
||||
expect(bodyDiv.style.scrollbarGutter).toBe('stable');
|
||||
|
||||
// Pin the `css={scrollBarStyles}` addition to header/footer directly (part
|
||||
// of the same secondary consistency measure as the `scrollbarGutter`
|
||||
// assertions above, not the clipping fix). This component carries
|
||||
// `/** @jsxImportSource @emotion/react */`, which makes
|
||||
// Babel route its `css` prop through Emotion's jsx runtime instead of
|
||||
// passing `css` straight through as an inert DOM attribute (the default in
|
||||
// this repo's Jest/Babel setup, which -- unlike the webpack/SWC build --
|
||||
// doesn't set `importSource: '@emotion/react'` globally). With the pragma
|
||||
// in place, an applied `css` prop is observable as a real, non-empty
|
||||
// className, so this assertion actually fails without the fix instead of
|
||||
// passing regardless of whether `scrollBarStyles` is wired up.
|
||||
//
|
||||
// Before `css={scrollBarStyles}` was added to header/footer, they had no
|
||||
// emotion-generated class at all (`className === ''`) while the body kept
|
||||
// its own -- so this fails pre-fix and passes post-fix.
|
||||
expect(headerDiv.className).not.toBe('');
|
||||
expect(headerDiv.className).toBe(bodyDiv.className);
|
||||
expect(footerDiv.className).toBe(bodyDiv.className);
|
||||
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
+3
-3
@@ -45,8 +45,8 @@ test('getCustomScrollBarSize measures the probe using the shared custom scrollba
|
||||
});
|
||||
|
||||
test('CUSTOM_SCROLLBAR_SIZE matches the custom scrollbar width rendered in the sticky table', () => {
|
||||
// useSticky.tsx's scrollBarStyles must stay in sync with this constant so
|
||||
// the sticky header's shrink amount always matches the body's real
|
||||
// scrollbar width.
|
||||
// useSticky.tsx's scrollBarStyles sets `::-webkit-scrollbar { width: ... }`
|
||||
// from this constant, so it must stay in sync with it or the real
|
||||
// scrollbar body/sizer render won't match what this constant claims.
|
||||
expect(CUSTOM_SCROLLBAR_SIZE).toBe(8);
|
||||
});
|
||||
|
||||
+97
-1
@@ -115,8 +115,21 @@ jest.mock('src/dashboard/components/nativeFilters/FilterBar', () => {
|
||||
MockFilterBar.displayName = 'MockFilterBar';
|
||||
return MockFilterBar;
|
||||
});
|
||||
// Exposes the sticky offset the builder hands to tab bars in the grid.
|
||||
jest.mock('src/dashboard/containers/DashboardGrid', () => {
|
||||
const MockDashboardGrid = () => <div data-test="mock-dashboard-grid" />;
|
||||
const { useContext } = jest.requireActual('react');
|
||||
const { StickyTabsOffsetContext } = jest.requireActual(
|
||||
'src/dashboard/components/gridComponents/TabsRenderer/StickyTabsOffsetContext',
|
||||
);
|
||||
const MockDashboardGrid = () => {
|
||||
const stickyTabsOffset = useContext(StickyTabsOffsetContext);
|
||||
return (
|
||||
<div
|
||||
data-test="mock-dashboard-grid"
|
||||
data-sticky-tabs-offset={stickyTabsOffset ?? 'none'}
|
||||
/>
|
||||
);
|
||||
};
|
||||
MockDashboardGrid.displayName = 'MockDashboardGrid';
|
||||
return MockDashboardGrid;
|
||||
});
|
||||
@@ -179,6 +192,89 @@ describe('DashboardBuilder', () => {
|
||||
expect(stickyContainer).toHaveClass('dashboard');
|
||||
});
|
||||
|
||||
// jsdom lays nothing out; report a height for the sticky header so the
|
||||
// offset handed to the grid is distinguishable from "no offset".
|
||||
function mockHeaderHeight(height: number) {
|
||||
return jest
|
||||
.spyOn(HTMLElement.prototype, 'getBoundingClientRect')
|
||||
.mockImplementation(function measure(this: HTMLElement) {
|
||||
const size =
|
||||
this.dataset.test === 'dashboard-header-wrapper' ? height : 0;
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: size,
|
||||
width: 0,
|
||||
height: size,
|
||||
toJSON: () => ({}),
|
||||
} as DOMRect;
|
||||
});
|
||||
}
|
||||
|
||||
test('hands the sticky header height to tab bars in the grid while viewing', async () => {
|
||||
const rectSpy = mockHeaderHeight(120);
|
||||
try {
|
||||
const { findByTestId } = setup();
|
||||
expect(await findByTestId('mock-dashboard-grid')).toHaveAttribute(
|
||||
'data-sticky-tabs-offset',
|
||||
'120',
|
||||
);
|
||||
} finally {
|
||||
rectSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('leaves tab bars in the grid unpinned in report mode (?standalone=3)', async () => {
|
||||
// Report screenshots of large dashboards are captured tile by tile while
|
||||
// scrolling the page; a pinned bar would repeat in every tile.
|
||||
const originalHref = window.location.href;
|
||||
window.history.replaceState({}, '', '/?standalone=3');
|
||||
const rectSpy = mockHeaderHeight(120);
|
||||
try {
|
||||
const { findByTestId } = setup();
|
||||
expect(await findByTestId('mock-dashboard-grid')).toHaveAttribute(
|
||||
'data-sticky-tabs-offset',
|
||||
'none',
|
||||
);
|
||||
} finally {
|
||||
rectSpy.mockRestore();
|
||||
window.history.replaceState({}, '', originalHref);
|
||||
}
|
||||
});
|
||||
|
||||
test('leaves tab bars in the grid unpinned while a chart is maximized', async () => {
|
||||
const rectSpy = mockHeaderHeight(120);
|
||||
try {
|
||||
const { findByTestId } = setup({
|
||||
dashboardState: { ...mockState.dashboardState, fullSizeChartId: 123 },
|
||||
});
|
||||
expect(await findByTestId('mock-dashboard-grid')).toHaveAttribute(
|
||||
'data-sticky-tabs-offset',
|
||||
'none',
|
||||
);
|
||||
} finally {
|
||||
rectSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('leaves tab bars in the grid unpinned in the mobile viewport', async () => {
|
||||
(useIsMobile as jest.Mock).mockReturnValue(true);
|
||||
const rectSpy = mockHeaderHeight(120);
|
||||
try {
|
||||
const { findByTestId } = setup();
|
||||
expect(await findByTestId('mock-dashboard-grid')).toHaveAttribute(
|
||||
'data-sticky-tabs-offset',
|
||||
'none',
|
||||
);
|
||||
} finally {
|
||||
rectSpy.mockRestore();
|
||||
(useIsMobile as jest.Mock).mockReturnValue(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('should add the "dashboard--editing" class if editMode=true', () => {
|
||||
const { getByTestId } = setup({
|
||||
dashboardState: { ...mockState.dashboardState, editMode: true },
|
||||
|
||||
@@ -87,6 +87,7 @@ import {
|
||||
} from 'src/dashboard/constants';
|
||||
import { selectCanRestoreDashboard } from 'src/features/versionHistory/canRestoreDashboard';
|
||||
import { selectIsDashboardVersionPreviewActive } from 'src/features/versionHistory/reducer';
|
||||
import { StickyTabsOffsetContext } from 'src/dashboard/components/gridComponents/TabsRenderer';
|
||||
import { getRootLevelTabsComponent, shouldFocusTabs } from './utils';
|
||||
import DashboardContainer from './DashboardContainer';
|
||||
import { useNativeFilters } from './state';
|
||||
@@ -490,8 +491,9 @@ const DashboardBuilder = () => {
|
||||
// always get the desktop layout -- matching the pre-existing behavior the
|
||||
// docs already promise for embedded dashboards.
|
||||
const standaloneMode = getUrlParam(URL_PARAMS.standalone);
|
||||
const isMobileViewport = useIsMobile();
|
||||
const isNotMobile =
|
||||
!useIsMobile() || standaloneMode !== DashboardStandaloneMode.None;
|
||||
!isMobileViewport || standaloneMode !== DashboardStandaloneMode.None;
|
||||
const [mobileFiltersOpen, setMobileFiltersOpen] = useState(false);
|
||||
|
||||
// Reset the drawer's open state when leaving mobile mode so it doesn't
|
||||
@@ -774,6 +776,17 @@ const DashboardBuilder = () => {
|
||||
? theme.sizeUnit * 4
|
||||
: theme.sizeUnit * 8;
|
||||
|
||||
// Tab bars nested in the grid pin just below the sticky header while the
|
||||
// page scrolls. Not in the mobile viewport, where the header scrolls away
|
||||
// and the mobile styling pins tab bars on its own; not in report mode,
|
||||
// whose tiled screenshots scroll the page and would capture a pinned bar
|
||||
// in every tile; and not while a chart is maximized, which sits inside its
|
||||
// own stacking context and must not be covered by a pinned bar.
|
||||
// (TabsRenderer itself opts out while editing, since drop targets rely on
|
||||
// document flow.)
|
||||
const stickyTabsOffset =
|
||||
isMobileViewport || isReport || fullSizeChartId ? undefined : barTopOffset;
|
||||
|
||||
const renderChild = useCallback(
|
||||
(adjustedWidth: number) => {
|
||||
const filterBarWidth = dashboardFiltersOpen
|
||||
@@ -976,7 +989,9 @@ const DashboardBuilder = () => {
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<DashboardContainer topLevelTabs={topLevelTabs} />
|
||||
<StickyTabsOffsetContext.Provider value={stickyTabsOffset}>
|
||||
<DashboardContainer topLevelTabs={topLevelTabs} />
|
||||
</StickyTabsOffsetContext.Provider>
|
||||
)
|
||||
) : (
|
||||
<Loading />
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { createContext } from 'react';
|
||||
|
||||
/**
|
||||
* Distance, in pixels, from the top of the viewport at which a tab bar
|
||||
* rendered inside the dashboard grid should stick while the page scrolls.
|
||||
*
|
||||
* The dashboard header (title plus top-level tabs) is itself sticky, so the
|
||||
* first level of nested tabs pins just below it. Each level of nested tabs
|
||||
* then adds its own tab bar height for the tabs it contains, so deeper tab
|
||||
* bars stack beneath the ones above them instead of overlapping.
|
||||
*
|
||||
* `undefined` disables sticky tab bars, which is the case while editing
|
||||
* (drag-and-drop targets are laid out in document flow) and in the mobile
|
||||
* consumption experience, which pins tab bars through its own styling.
|
||||
*/
|
||||
export const StickyTabsOffsetContext = createContext<number | undefined>(
|
||||
undefined,
|
||||
);
|
||||
+169
-2
@@ -16,8 +16,15 @@
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { fireEvent, render, screen } from 'spec/helpers/testing-library';
|
||||
import {
|
||||
act,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
sleep,
|
||||
} from 'spec/helpers/testing-library';
|
||||
import TabsRenderer, { TabItem, TabsRendererProps } from './TabsRenderer';
|
||||
import { StickyTabsOffsetContext } from './StickyTabsOffsetContext';
|
||||
|
||||
const mockTabItems: TabItem[] = [
|
||||
{
|
||||
@@ -240,7 +247,7 @@ describe('TabsRenderer', () => {
|
||||
expect(screen.queryByText('Tab 2 Content')).not.toBeInTheDocument(); // Not active
|
||||
});
|
||||
|
||||
test('drags from the tab title and shows the drag indicator only then', () => {
|
||||
test('drags from the tab title and shows the drag indicator only then', async () => {
|
||||
render(<TabsRenderer {...draggableTabProps} />);
|
||||
const container = screen.getByTestId('dashboard-component-tabs');
|
||||
const title = container.querySelector('textarea') as HTMLTextAreaElement;
|
||||
@@ -263,5 +270,165 @@ describe('TabsRenderer', () => {
|
||||
expect(container).toHaveStyleRule('cursor', 'move', {
|
||||
target: '.dragdroppable-tab *',
|
||||
});
|
||||
|
||||
// Release the pointer so the drag does not outlive this test. dnd-kit
|
||||
// keeps swallowing clicks on the shared document for 50ms after a drag
|
||||
// ends, which would eat the tab click of whichever test runs next.
|
||||
fireEvent.pointerUp(document, { button: 0, isPrimary: true, clientX: 50 });
|
||||
await sleep(60);
|
||||
});
|
||||
|
||||
// jsdom's cascade ignores specificity, so assert on the emotion rule rather
|
||||
// than the computed style, which antd's own `position: relative` would win
|
||||
const TAB_BAR = { target: /> ?\.ant-tabs ?> ?\.ant-tabs-nav$/ };
|
||||
|
||||
test('pins the tab bar below the offset supplied by the dashboard', () => {
|
||||
render(
|
||||
<StickyTabsOffsetContext.Provider value={64}>
|
||||
<TabsRenderer {...mockProps} />
|
||||
</StickyTabsOffsetContext.Provider>,
|
||||
);
|
||||
const container = screen.getByTestId('dashboard-component-tabs');
|
||||
|
||||
expect(container).toHaveStyleRule('position', 'sticky', TAB_BAR);
|
||||
expect(container).toHaveStyleRule('top', '64px', TAB_BAR);
|
||||
});
|
||||
|
||||
test('leaves the tab bar in document flow without a dashboard offset', () => {
|
||||
render(<TabsRenderer {...mockProps} />);
|
||||
const container = screen.getByTestId('dashboard-component-tabs');
|
||||
|
||||
expect(container).not.toHaveStyleRule('position', 'sticky', TAB_BAR);
|
||||
});
|
||||
|
||||
test('leaves the tab bar in document flow in edit mode', () => {
|
||||
render(
|
||||
<StickyTabsOffsetContext.Provider value={64}>
|
||||
<TabsRenderer {...mockProps} editMode />
|
||||
</StickyTabsOffsetContext.Provider>,
|
||||
);
|
||||
const container = screen.getByTestId('dashboard-component-tabs');
|
||||
|
||||
expect(container).not.toHaveStyleRule('position', 'sticky', TAB_BAR);
|
||||
});
|
||||
|
||||
// Reports the offset this tab set hands to tab sets nested inside it.
|
||||
const nestedTabItems: TabItem[] = [
|
||||
{
|
||||
...mockTabItems[0],
|
||||
children: (
|
||||
<StickyTabsOffsetContext.Consumer>
|
||||
{offset => <div data-test="nested-offset">{offset}</div>}
|
||||
</StickyTabsOffsetContext.Consumer>
|
||||
),
|
||||
},
|
||||
mockTabItems[1],
|
||||
];
|
||||
|
||||
test('stacks nested tab bars beneath its own tab bar', () => {
|
||||
// jsdom lays nothing out, so give the tab bar a height to add up
|
||||
const heightSpy = jest
|
||||
.spyOn(HTMLElement.prototype, 'offsetHeight', 'get')
|
||||
.mockReturnValue(40);
|
||||
|
||||
try {
|
||||
render(
|
||||
<StickyTabsOffsetContext.Provider value={64}>
|
||||
<TabsRenderer {...mockProps} tabItems={nestedTabItems} />
|
||||
</StickyTabsOffsetContext.Provider>,
|
||||
);
|
||||
|
||||
expect(screen.getByTestId('nested-offset')).toHaveTextContent('104');
|
||||
} finally {
|
||||
heightSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('restacks nested tab bars when its own tab bar reflows', () => {
|
||||
// The shared jsdom shim never fires its callback, so stand in an observer
|
||||
// that hands the callback back to the test. The component re-measures the
|
||||
// element rather than reading the entries, so the height comes from the
|
||||
// spy below; the callback only needs to run.
|
||||
const observerCallbacks: ResizeObserverCallback[] = [];
|
||||
const RealResizeObserver = window.ResizeObserver;
|
||||
window.ResizeObserver = class {
|
||||
constructor(callback: ResizeObserverCallback) {
|
||||
observerCallbacks.push(callback);
|
||||
}
|
||||
|
||||
observe() {}
|
||||
|
||||
unobserve() {}
|
||||
|
||||
disconnect() {}
|
||||
} as unknown as typeof ResizeObserver;
|
||||
const heightSpy = jest
|
||||
.spyOn(HTMLElement.prototype, 'offsetHeight', 'get')
|
||||
.mockReturnValue(40);
|
||||
|
||||
try {
|
||||
render(
|
||||
<StickyTabsOffsetContext.Provider value={64}>
|
||||
<TabsRenderer {...mockProps} tabItems={nestedTabItems} />
|
||||
</StickyTabsOffsetContext.Provider>,
|
||||
);
|
||||
expect(screen.getByTestId('nested-offset')).toHaveTextContent('104');
|
||||
|
||||
// The bar grows -- labels wrap on a narrow viewport, a webfont lands --
|
||||
// and the tab set below it has to move down by the same amount.
|
||||
heightSpy.mockReturnValue(80);
|
||||
act(() => {
|
||||
observerCallbacks.forEach(callback =>
|
||||
callback([], {} as ResizeObserver),
|
||||
);
|
||||
});
|
||||
|
||||
expect(screen.getByTestId('nested-offset')).toHaveTextContent('144');
|
||||
} finally {
|
||||
heightSpy.mockRestore();
|
||||
window.ResizeObserver = RealResizeObserver;
|
||||
}
|
||||
});
|
||||
|
||||
// Switching tabs while the bar is pinned: the page scrolls so the tab set
|
||||
// starts where its bar is pinned, mirroring the top-level tabs' jump to top
|
||||
function renderPinnedTabSet(containerTop: number, offset?: number) {
|
||||
const scrollTo = jest.spyOn(window, 'scrollTo').mockImplementation();
|
||||
const rectSpy = jest
|
||||
.spyOn(HTMLElement.prototype, 'getBoundingClientRect')
|
||||
.mockReturnValue({ top: containerTop } as DOMRect);
|
||||
const scrollYStub = jest.replaceProperty(window, 'scrollY', 500);
|
||||
render(
|
||||
<StickyTabsOffsetContext.Provider value={offset}>
|
||||
<TabsRenderer {...mockProps} />
|
||||
</StickyTabsOffsetContext.Provider>,
|
||||
);
|
||||
fireEvent.click(screen.getByText('Tab 2').closest('[role="tab"]')!);
|
||||
rectSpy.mockRestore();
|
||||
scrollYStub.restore();
|
||||
return scrollTo;
|
||||
}
|
||||
|
||||
test('scrolls a pinned tab set back to its top when switching tabs', () => {
|
||||
// the tab set's top is 200px above the viewport, so the bar is pinned
|
||||
const scrollTo = renderPinnedTabSet(-200, 64);
|
||||
|
||||
expect(scrollTo).toHaveBeenCalledWith(window.scrollX, 500 - 200 - 64);
|
||||
scrollTo.mockRestore();
|
||||
});
|
||||
|
||||
test('leaves the page alone when the tab bar is not pinned', () => {
|
||||
// the tab set starts below where its bar would pin, so it is in flow
|
||||
const scrollTo = renderPinnedTabSet(300, 64);
|
||||
|
||||
expect(scrollTo).not.toHaveBeenCalled();
|
||||
scrollTo.mockRestore();
|
||||
});
|
||||
|
||||
test('leaves the page alone without a dashboard offset', () => {
|
||||
const scrollTo = renderPinnedTabSet(-200);
|
||||
|
||||
expect(scrollTo).not.toHaveBeenCalled();
|
||||
scrollTo.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
+132
-47
@@ -22,6 +22,8 @@ import {
|
||||
ReactElement,
|
||||
RefObject,
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from 'react';
|
||||
@@ -46,11 +48,33 @@ import {
|
||||
import HoverMenu from '../../menu/HoverMenu';
|
||||
import DragHandle from '../../dnd/DragHandle';
|
||||
import DeleteComponentButton from '../../DeleteComponentButton';
|
||||
import { StickyTabsOffsetContext } from './StickyTabsOffsetContext';
|
||||
|
||||
const StyledTabsContainer = styled.div<{ isDragging?: boolean }>`
|
||||
// @z-index-above-dashboard-charts: above chart content, below the sticky
|
||||
// dashboard header (99) and the filter bar (11)
|
||||
const STICKY_TAB_BAR_Z_INDEX = 10;
|
||||
|
||||
const StyledTabsContainer = styled.div<{
|
||||
isDragging?: boolean;
|
||||
stickyTop?: number;
|
||||
}>`
|
||||
width: 100%;
|
||||
background-color: ${({ theme }) => theme.colorBgContainer};
|
||||
|
||||
/* Pin this component's own tab bar (direct child only, so nested tab
|
||||
sets keep their own offsets) below the sticky dashboard header while
|
||||
its content scrolls. */
|
||||
${({ theme, stickyTop }) =>
|
||||
stickyTop !== undefined &&
|
||||
css`
|
||||
& > .ant-tabs > .ant-tabs-nav {
|
||||
position: sticky;
|
||||
top: ${stickyTop}px;
|
||||
z-index: ${STICKY_TAB_BAR_Z_INDEX};
|
||||
background-color: ${theme.colorBgContainer};
|
||||
}
|
||||
`}
|
||||
|
||||
& .dashboard-component-tabs-content {
|
||||
height: 100%;
|
||||
}
|
||||
@@ -231,6 +255,59 @@ const TabsRenderer = memo<TabsRendererProps>(
|
||||
}) => {
|
||||
const [activeId, setActiveId] = useState<string | null>(null);
|
||||
|
||||
// Sticky tab bars only apply in view mode: while editing, drag-and-drop
|
||||
// targets and hover menus are positioned against the tab bar's place in
|
||||
// document flow.
|
||||
const parentStickyOffset = useContext(StickyTabsOffsetContext);
|
||||
const stickyTop = editMode ? undefined : parentStickyOffset;
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const [tabBarHeight, setTabBarHeight] = useState(0);
|
||||
|
||||
// Keyed on whether the bar is pinned rather than on the offset itself:
|
||||
// the header re-measuring on every resize would otherwise tear down and
|
||||
// recreate the observer in every tab set on the page.
|
||||
const isPinned = stickyTop !== undefined;
|
||||
useEffect(() => {
|
||||
// Direct-child selector, so nested tab sets' bars are never matched.
|
||||
const tabBar = isPinned
|
||||
? containerRef.current?.querySelector<HTMLElement>(
|
||||
':scope > .ant-tabs > .ant-tabs-nav',
|
||||
)
|
||||
: null;
|
||||
if (!tabBar) {
|
||||
return undefined;
|
||||
}
|
||||
const measure = () => setTabBarHeight(tabBar.offsetHeight);
|
||||
measure();
|
||||
// Matches the sticky header's own guard in DashboardBuilder, for
|
||||
// environments without ResizeObserver: the bar still pins, it just
|
||||
// keeps the height measured at mount.
|
||||
if (!global.hasOwnProperty('ResizeObserver')) {
|
||||
return undefined;
|
||||
}
|
||||
const observer = new ResizeObserver(measure);
|
||||
observer.observe(tabBar);
|
||||
return () => observer.disconnect();
|
||||
}, [isPinned]);
|
||||
|
||||
// Tabs nested inside this one stack their bar beneath ours.
|
||||
const childStickyOffset =
|
||||
stickyTop === undefined ? undefined : stickyTop + tabBarHeight;
|
||||
|
||||
// Counterpart of the top-level tabs, which scroll the page to the top on
|
||||
// every switch (DashboardBuilder.handleChangeTab). With this bar pinned,
|
||||
// a switch would otherwise land the reader partway down the new tab's
|
||||
// content; bring the tab set back to where its bar is pinned instead.
|
||||
const scrollPinnedTabSetToTop = () => {
|
||||
if (stickyTop === undefined || !containerRef.current) {
|
||||
return;
|
||||
}
|
||||
const { top } = containerRef.current.getBoundingClientRect();
|
||||
if (top < stickyTop) {
|
||||
window.scrollTo(window.scrollX, window.scrollY + top - stickyTop);
|
||||
}
|
||||
};
|
||||
|
||||
// Use ref to always have access to the current tabIds in callbacks
|
||||
const tabIdsRef = useRef(tabIds);
|
||||
tabIdsRef.current = tabIds;
|
||||
@@ -267,9 +344,11 @@ const TabsRenderer = memo<TabsRendererProps>(
|
||||
|
||||
return (
|
||||
<StyledTabsContainer
|
||||
ref={containerRef}
|
||||
className="dashboard-component dashboard-component-tabs"
|
||||
data-test="dashboard-component-tabs"
|
||||
isDragging={isDragging}
|
||||
stickyTop={stickyTop}
|
||||
>
|
||||
{editMode && renderHoverMenu && tabsDragSourceRef && (
|
||||
<HoverMenu innerRef={tabsDragSourceRef} position="left">
|
||||
@@ -278,53 +357,59 @@ const TabsRenderer = memo<TabsRendererProps>(
|
||||
</HoverMenu>
|
||||
)}
|
||||
|
||||
<LineEditableTabs
|
||||
id={tabsComponent.id}
|
||||
activeKey={activeKey}
|
||||
onChange={key => {
|
||||
if (typeof key === 'string') {
|
||||
const tabIndex = tabIds.indexOf(key);
|
||||
if (tabIndex !== -1) handleClickTab(tabIndex);
|
||||
}
|
||||
}}
|
||||
onEdit={handleEdit}
|
||||
data-test="nav-list"
|
||||
type={editMode ? 'editable-card' : 'card'}
|
||||
items={tabItems}
|
||||
tabBarStyle={{ paddingLeft: tabBarPaddingLeft }}
|
||||
fullHeight
|
||||
{...(editMode && {
|
||||
renderTabBar: (tabBarProps, DefaultTabBar) => (
|
||||
<DndContext
|
||||
key={tabIds.join('-')}
|
||||
sensors={[sensor]}
|
||||
onDragStart={onDragStart}
|
||||
onDragEnd={onDragEnd}
|
||||
onDragCancel={onDragCancel}
|
||||
collisionDetection={closestCenter}
|
||||
>
|
||||
<SortableContext
|
||||
items={tabIds}
|
||||
strategy={horizontalListSortingStrategy}
|
||||
<StickyTabsOffsetContext.Provider value={childStickyOffset}>
|
||||
<LineEditableTabs
|
||||
id={tabsComponent.id}
|
||||
activeKey={activeKey}
|
||||
onChange={key => {
|
||||
if (typeof key === 'string') {
|
||||
const tabIndex = tabIds.indexOf(key);
|
||||
if (tabIndex !== -1) {
|
||||
handleClickTab(tabIndex);
|
||||
scrollPinnedTabSetToTop();
|
||||
}
|
||||
}
|
||||
}}
|
||||
onEdit={handleEdit}
|
||||
data-test="nav-list"
|
||||
type={editMode ? 'editable-card' : 'card'}
|
||||
items={tabItems}
|
||||
tabBarStyle={{ paddingLeft: tabBarPaddingLeft }}
|
||||
fullHeight
|
||||
{...(editMode && {
|
||||
renderTabBar: (tabBarProps, DefaultTabBar) => (
|
||||
<DndContext
|
||||
key={tabIds.join('-')}
|
||||
sensors={[sensor]}
|
||||
onDragStart={onDragStart}
|
||||
onDragEnd={onDragEnd}
|
||||
onDragCancel={onDragCancel}
|
||||
collisionDetection={closestCenter}
|
||||
>
|
||||
<DefaultTabBar {...tabBarProps}>
|
||||
{(node: React.ReactElement) => (
|
||||
<DraggableTabNode
|
||||
{...(node as React.ReactElement<DraggableTabNodeProps>)
|
||||
.props}
|
||||
key={node.key}
|
||||
data-node-key={node.key as string}
|
||||
disabled={isEditingTabTitle}
|
||||
>
|
||||
{node}
|
||||
</DraggableTabNode>
|
||||
)}
|
||||
</DefaultTabBar>
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
),
|
||||
})}
|
||||
/>
|
||||
<SortableContext
|
||||
items={tabIds}
|
||||
strategy={horizontalListSortingStrategy}
|
||||
>
|
||||
<DefaultTabBar {...tabBarProps}>
|
||||
{(node: React.ReactElement) => (
|
||||
<DraggableTabNode
|
||||
{...(
|
||||
node as React.ReactElement<DraggableTabNodeProps>
|
||||
).props}
|
||||
key={node.key}
|
||||
data-node-key={node.key as string}
|
||||
disabled={isEditingTabTitle}
|
||||
>
|
||||
{node}
|
||||
</DraggableTabNode>
|
||||
)}
|
||||
</DefaultTabBar>
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
),
|
||||
})}
|
||||
/>
|
||||
</StickyTabsOffsetContext.Provider>
|
||||
</StyledTabsContainer>
|
||||
);
|
||||
},
|
||||
|
||||
@@ -18,3 +18,4 @@
|
||||
*/
|
||||
export { default } from './TabsRenderer';
|
||||
export type { TabsRendererProps, TabItem, TabsComponent } from './TabsRenderer';
|
||||
export { StickyTabsOffsetContext } from './StickyTabsOffsetContext';
|
||||
|
||||
+177
-2
@@ -41,6 +41,38 @@ const callbackRef: {
|
||||
| null;
|
||||
} = { current: null };
|
||||
|
||||
// Real DropdownContainer partitions `items` into a visible main row and an
|
||||
// overflow slice by array index (`items.slice(0, overflowingIndex)` /
|
||||
// `items.slice(overflowingIndex)`), computed fresh every render in a
|
||||
// `useLayoutEffect` from live DOM measurements (DropdownContainer.tsx
|
||||
// lines ~166-234). It then reports that same partition to its parent
|
||||
// *separately*, one render later, via a plain `useEffect`
|
||||
// (onOverflowingStateChange, lines ~236-243) — a different effect phase
|
||||
// than the one that produced the partition it's reporting.
|
||||
//
|
||||
// This mock models both halves of that architecture instead of collapsing
|
||||
// them into one: `mockOverflowingIndex` stands in for DropdownContainer's
|
||||
// own always-fresh partition boundary — every render reads it and slices
|
||||
// `props.items` accordingly, exactly like production. `callbackRef` /
|
||||
// `fireOverflow` stand in for the separate, asynchronously delivered
|
||||
// onOverflowingStateChange report that FilterControls actually listens to
|
||||
// and mirrors into its own `overflowedIds` state. Nothing here auto-syncs
|
||||
// the two: a test can move `mockOverflowingIndex` (DropdownContainer having
|
||||
// *already* recomputed a new partition) without calling `fireOverflow`
|
||||
// again (its useEffect not having reported that new partition to the
|
||||
// parent yet) — reproducing the one-render lag that exists in production
|
||||
// between DropdownContainer's synchronous useLayoutEffect and its
|
||||
// asynchronous useEffect, rather than asserting the duplicate by
|
||||
// construction.
|
||||
//
|
||||
// Mirrors production's own sentinel exactly: -1 means "nothing overflows"
|
||||
// (DropdownContainer.tsx's `overflowingIndex` default), any other value is
|
||||
// the boundary index passed to `items.slice(0, n)` / `items.slice(n)`. A
|
||||
// bare `0` here means "everything overflows" (empty main row), so tests
|
||||
// that need that state set it explicitly rather than relying on the
|
||||
// default reading as "no overflow" by analogy with the real sentinel.
|
||||
let mockOverflowingIndex = -1;
|
||||
|
||||
// Mock the DropdownContainer subpath rather than the barrel
|
||||
// `@superset-ui/core/components` — mocking the barrel triggers a
|
||||
// circular re-export chain at requireActual time
|
||||
@@ -57,10 +89,18 @@ jest.mock('@superset-ui/core/components/DropdownContainer', () => {
|
||||
open: jest.fn(),
|
||||
close: jest.fn(),
|
||||
}));
|
||||
const notOverflowed =
|
||||
mockOverflowingIndex !== -1
|
||||
? props.items.slice(0, mockOverflowingIndex)
|
||||
: props.items;
|
||||
const overflowed =
|
||||
mockOverflowingIndex !== -1
|
||||
? props.items.slice(mockOverflowingIndex)
|
||||
: [];
|
||||
return (
|
||||
<div data-test="dropdown-container-mock">
|
||||
<div data-test="dropdown-items">
|
||||
{props.items.map((item: DropdownItem) => (
|
||||
{notOverflowed.map((item: DropdownItem) => (
|
||||
<div key={item.id} data-test="dropdown-item">
|
||||
{item.element}
|
||||
</div>
|
||||
@@ -74,7 +114,7 @@ jest.mock('@superset-ui/core/components/DropdownContainer', () => {
|
||||
</div>
|
||||
{props.dropdownContent && (
|
||||
<div data-test="dropdown-content-mock">
|
||||
{props.dropdownContent([])}
|
||||
{props.dropdownContent(overflowed)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -182,6 +222,7 @@ const fireOverflow = (overflowed: string[], notOverflowed: string[]) => {
|
||||
beforeEach(() => {
|
||||
dropdownContainerProps.length = 0;
|
||||
callbackRef.current = null;
|
||||
mockOverflowingIndex = -1;
|
||||
});
|
||||
|
||||
test('horizontal FilterControls hands every filter to DropdownContainer as an item', async () => {
|
||||
@@ -274,6 +315,137 @@ test('firing overflow with no active values keeps trigger count at 0 but supplie
|
||||
expect(latestProps().dropdownTriggerCount).toBe(0);
|
||||
});
|
||||
|
||||
// Cross-filter chips are keyed by `${name}${emitterId}` (see FilterControls.tsx's
|
||||
// `items` memo) and sourced from `crossFiltersSelector`, which reads
|
||||
// `dashboardState.sliceIds` + `dashboardLayout.present` (for the chart's name) +
|
||||
// `dataMask` (for the emitted filter's column/value) — independent of the native
|
||||
// filter config used by `buildHorizontalState` above.
|
||||
const CROSS_FILTER_CHART_ID = 85;
|
||||
const CROSS_FILTER_CHART_NAME = 'Products Sold By Product Line';
|
||||
const CROSS_FILTER_ITEM_ID = `${CROSS_FILTER_CHART_NAME}${CROSS_FILTER_CHART_ID}`;
|
||||
|
||||
const buildStateWithOneCrossFilter = () => ({
|
||||
...buildHorizontalState([]),
|
||||
dashboardState: {
|
||||
sliceIds: [CROSS_FILTER_CHART_ID],
|
||||
activeTabs: ['ROOT_ID'],
|
||||
},
|
||||
dashboardLayout: {
|
||||
present: {
|
||||
ROOT_ID: {
|
||||
type: 'ROOT',
|
||||
id: 'ROOT_ID',
|
||||
children: [`CHART-${CROSS_FILTER_CHART_ID}`],
|
||||
},
|
||||
[`CHART-${CROSS_FILTER_CHART_ID}`]: {
|
||||
type: 'CHART',
|
||||
id: `CHART-${CROSS_FILTER_CHART_ID}`,
|
||||
parents: ['ROOT_ID'],
|
||||
meta: {
|
||||
chartId: CROSS_FILTER_CHART_ID,
|
||||
sliceName: CROSS_FILTER_CHART_NAME,
|
||||
},
|
||||
},
|
||||
},
|
||||
past: [],
|
||||
future: [],
|
||||
},
|
||||
dataMask: {
|
||||
[CROSS_FILTER_CHART_ID]: {
|
||||
id: CROSS_FILTER_CHART_ID,
|
||||
filterState: {
|
||||
value: 'Classic Cars',
|
||||
filters: { product_line: 'Classic Cars' },
|
||||
},
|
||||
extraFormData: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const crossFilterControlsElement = (
|
||||
<FilterControls
|
||||
dataMaskSelected={{}}
|
||||
onFilterSelectionChange={jest.fn()}
|
||||
onPendingCustomizationDataMaskChange={jest.fn()}
|
||||
chartCustomizationValues={[]}
|
||||
/>
|
||||
);
|
||||
|
||||
const countChipCopies = async () => {
|
||||
const mainRow = within(
|
||||
await within(document.body).findByTestId('dropdown-items'),
|
||||
).queryAllByText(CROSS_FILTER_CHART_NAME);
|
||||
const popover = within(
|
||||
await within(document.body).findByTestId('dropdown-content-mock'),
|
||||
).queryAllByText(CROSS_FILTER_CHART_NAME);
|
||||
return mainRow.length + popover.length;
|
||||
};
|
||||
|
||||
test('a cross-filter chip DropdownContainer has already stopped overflowing does not also render in the popover', async () => {
|
||||
// Regression guard for the FilterBar duplicate-chip bug. This drives the
|
||||
// exact two-channel desync described in RCA.md: DropdownContainer's own
|
||||
// main-row partition (modeled here by `mockOverflowingIndex`, standing in
|
||||
// for its real useLayoutEffect-computed overflowingIndex) updates
|
||||
// synchronously and independently of the separate, asynchronous
|
||||
// onOverflowingStateChange report FilterControls mirrors into its
|
||||
// `overflowedIds` state (driven here by the real onOverflowingStateChange
|
||||
// callback via `fireOverflow`). Moving one without the other reproduces
|
||||
// the one-render lag that exists in production between DropdownContainer's
|
||||
// useLayoutEffect (immediate) and its useEffect (runs one commit later).
|
||||
const { rerender } = render(crossFilterControlsElement, {
|
||||
useRedux: true,
|
||||
useRouter: true,
|
||||
initialState: buildStateWithOneCrossFilter(),
|
||||
});
|
||||
|
||||
await waitFor(() => expect(callbackRef.current).toBeTruthy());
|
||||
await waitFor(() =>
|
||||
expect(latestProps().items.map((i: DropdownItem) => i.id)).toContain(
|
||||
CROSS_FILTER_ITEM_ID,
|
||||
),
|
||||
);
|
||||
|
||||
// Step 1 — settled baseline: DropdownContainer's own partition
|
||||
// (mockOverflowingIndex = 0 — the sole item overflows) already excludes
|
||||
// the cross filter from the main row, and its onOverflowingStateChange
|
||||
// report agrees (fired via fireOverflow). Prove the two channels are
|
||||
// consistent and there is exactly one copy of the chip before touching
|
||||
// anything.
|
||||
mockOverflowingIndex = 0;
|
||||
fireOverflow([CROSS_FILTER_ITEM_ID], []);
|
||||
await waitFor(async () => expect(await countChipCopies()).toBe(1));
|
||||
|
||||
// Step 2 — DropdownContainer, on its own, recomputes a new partition
|
||||
// putting the cross filter back in the main row (e.g. more horizontal
|
||||
// space became available) — a plain rerender is enough to make the mock
|
||||
// re-read the moved `mockOverflowingIndex`, exactly like DropdownContainer
|
||||
// re-running its useLayoutEffect on a real resize. Deliberately do NOT
|
||||
// call fireOverflow again: production's matching useEffect runs strictly
|
||||
// after the layout effect that produced this new partition, so at this
|
||||
// point FilterControls has not been told about it yet.
|
||||
mockOverflowingIndex = 1;
|
||||
rerender(crossFilterControlsElement);
|
||||
|
||||
// DropdownContainer's fresh partition already shows the chip in the main
|
||||
// row this render — confirms the mock's synchronous half actually moved,
|
||||
// not just that nothing changed.
|
||||
await waitFor(async () => {
|
||||
const mainRow = within(
|
||||
await within(document.body).findByTestId('dropdown-items'),
|
||||
).queryAllByText(CROSS_FILTER_CHART_NAME);
|
||||
expect(mainRow.length).toBe(1);
|
||||
});
|
||||
|
||||
// FilterControls' overflowedIds state is still stale here (no fireOverflow
|
||||
// call happened for this new partition) — but the popover's content no
|
||||
// longer reads from that state. It's built from the same-render
|
||||
// `overflowedItems` argument DropdownContainer passes into dropdownContent,
|
||||
// which already excludes this chip (it just moved to the main row this
|
||||
// same render), so the popover correctly omits it despite the parent's
|
||||
// stale mirrored state disagreeing. Total count stays at one copy.
|
||||
expect(await countChipCopies()).toBe(1);
|
||||
});
|
||||
|
||||
test('all 12 overflowed filters are reachable through dropdownContent', async () => {
|
||||
// Substitutes for the disabled Cypress "scroll within overflow" assertion:
|
||||
// jsdom has no real layout/scrolling, so we instead prove every overflowed
|
||||
@@ -282,6 +454,9 @@ test('all 12 overflowed filters are reachable through dropdownContent', async ()
|
||||
createSelectNativeFilter(`NATIVE_FILTER-${i + 1}`, `filter_${i + 1}`),
|
||||
);
|
||||
|
||||
// All 12 filters overflow — DropdownContainer's own partition puts
|
||||
// nothing in the main row (mirrors real production 0-boundary).
|
||||
mockOverflowingIndex = 0;
|
||||
renderHorizontal(filters, buildDataMaskSelected(filters));
|
||||
|
||||
await waitFor(() => expect(callbackRef.current).toBeTruthy());
|
||||
|
||||
+60
-19
@@ -58,6 +58,7 @@ import {
|
||||
import { FilterBarOrientation, RootState } from 'src/dashboard/types';
|
||||
import {
|
||||
DropdownContainer,
|
||||
type DropdownItem,
|
||||
type DropdownRef as DropdownContainerRef,
|
||||
Typography,
|
||||
} from '@superset-ui/core/components';
|
||||
@@ -447,6 +448,14 @@ const FilterControls: FC<FilterControlsProps> = ({
|
||||
return [...activeOverflowedFilters, ...overflowedCrossFilters];
|
||||
}, [overflowedCrossFilters, overflowedFiltersInScope]);
|
||||
|
||||
const overflowedCustomizationsInScope = useMemo(
|
||||
() =>
|
||||
customizationsInScope.filter(({ id }) =>
|
||||
overflowedIds?.includes(`chart-customization-${id}`),
|
||||
),
|
||||
[customizationsInScope, overflowedIds],
|
||||
);
|
||||
|
||||
const rendererCrossFilter = useCallback(
|
||||
(
|
||||
crossFilter: CrossFilterIndicator,
|
||||
@@ -610,28 +619,56 @@ const FilterControls: FC<FilterControlsProps> = ({
|
||||
dropdownContent={
|
||||
overflowedFiltersInScope.length ||
|
||||
overflowedCrossFilters.length ||
|
||||
overflowedCustomizationsInScope.length ||
|
||||
(filtersOutOfScope.length && showCollapsePanel) ||
|
||||
(customizationsOutOfScope.length && showCustomizationCollapsePanel)
|
||||
? () => (
|
||||
<>
|
||||
<FiltersDropdownContent
|
||||
overflowedCrossFilters={overflowedCrossFilters}
|
||||
filtersInScope={overflowedFiltersInScope}
|
||||
filtersOutOfScope={filtersOutOfScope}
|
||||
renderer={renderer}
|
||||
rendererCrossFilter={rendererCrossFilter}
|
||||
showCollapsePanel={showCollapsePanel}
|
||||
forceRenderOutOfScope={hasRequiredFirst}
|
||||
/>
|
||||
{showCustomizationCollapsePanel && (
|
||||
<CustomizationsOutOfScopeCollapsible
|
||||
customizationsOutOfScope={customizationsOutOfScope}
|
||||
renderer={customizationRenderer}
|
||||
forceRender={false}
|
||||
? (overflowedItems: DropdownItem[]) => {
|
||||
// Which ids are overflowed comes from DropdownContainer's own
|
||||
// fresh, synchronous partition of `items` (the argument it
|
||||
// passes here), not from `overflowedIds` state — that state
|
||||
// only updates one render later via onOverflowingStateChange,
|
||||
// so using it here could show a filter here that
|
||||
// DropdownContainer's *own* main row, computed this same
|
||||
// render, has already stopped excluding (duplicate chip).
|
||||
const overflowedItemIds = new Set(
|
||||
overflowedItems.map(item => item.id),
|
||||
);
|
||||
const freshOverflowedFiltersInScope = filtersInScope.filter(
|
||||
({ id }) => overflowedItemIds.has(id),
|
||||
);
|
||||
const freshOverflowedCrossFilters =
|
||||
selectedCrossFilters.filter(({ emitterId, name }) =>
|
||||
overflowedItemIds.has(`${name}${emitterId}`),
|
||||
);
|
||||
const freshOverflowedCustomizationsInScope =
|
||||
customizationsInScope.filter(({ id }) =>
|
||||
overflowedItemIds.has(`chart-customization-${id}`),
|
||||
);
|
||||
return (
|
||||
<>
|
||||
<FiltersDropdownContent
|
||||
overflowedCrossFilters={freshOverflowedCrossFilters}
|
||||
filtersInScope={freshOverflowedFiltersInScope}
|
||||
filtersOutOfScope={filtersOutOfScope}
|
||||
overflowedCustomizationsInScope={
|
||||
freshOverflowedCustomizationsInScope
|
||||
}
|
||||
renderer={renderer}
|
||||
rendererCrossFilter={rendererCrossFilter}
|
||||
customizationRenderer={customizationRenderer}
|
||||
showCollapsePanel={showCollapsePanel}
|
||||
forceRenderOutOfScope={hasRequiredFirst}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
{showCustomizationCollapsePanel && (
|
||||
<CustomizationsOutOfScopeCollapsible
|
||||
customizationsOutOfScope={customizationsOutOfScope}
|
||||
renderer={customizationRenderer}
|
||||
forceRender={false}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
forceRender={hasRequiredFirst}
|
||||
@@ -655,6 +692,10 @@ const FilterControls: FC<FilterControlsProps> = ({
|
||||
activeOverflowedFiltersInScope,
|
||||
overflowedFiltersInScope,
|
||||
overflowedCrossFilters,
|
||||
overflowedCustomizationsInScope,
|
||||
filtersInScope,
|
||||
selectedCrossFilters,
|
||||
customizationsInScope,
|
||||
filtersOutOfScope,
|
||||
showCollapsePanel,
|
||||
customizationsOutOfScope,
|
||||
|
||||
+20
-1
@@ -18,7 +18,12 @@
|
||||
*/
|
||||
|
||||
import { ReactNode } from 'react';
|
||||
import { Divider, Filter } from '@superset-ui/core';
|
||||
import {
|
||||
ChartCustomization,
|
||||
ChartCustomizationDivider,
|
||||
Divider,
|
||||
Filter,
|
||||
} from '@superset-ui/core';
|
||||
import { css, SupersetTheme } from '@apache-superset/core/theme';
|
||||
import { FilterBarOrientation } from 'src/dashboard/types';
|
||||
import { FiltersOutOfScopeCollapsible } from '../FiltersOutOfScopeCollapsible';
|
||||
@@ -28,12 +33,20 @@ export interface FiltersDropdownContentProps {
|
||||
overflowedCrossFilters: CrossFilterIndicator[];
|
||||
filtersInScope: (Filter | Divider)[];
|
||||
filtersOutOfScope: (Filter | Divider)[];
|
||||
overflowedCustomizationsInScope?: (
|
||||
| ChartCustomization
|
||||
| ChartCustomizationDivider
|
||||
)[];
|
||||
renderer: (filter: Filter | Divider, index: number) => ReactNode;
|
||||
rendererCrossFilter: (
|
||||
crossFilter: CrossFilterIndicator,
|
||||
orientation: FilterBarOrientation.Vertical,
|
||||
last: CrossFilterIndicator,
|
||||
) => ReactNode;
|
||||
customizationRenderer?: (
|
||||
item: ChartCustomization | ChartCustomizationDivider,
|
||||
index: number,
|
||||
) => ReactNode;
|
||||
showCollapsePanel?: boolean;
|
||||
forceRenderOutOfScope?: boolean;
|
||||
}
|
||||
@@ -42,8 +55,10 @@ export const FiltersDropdownContent = ({
|
||||
overflowedCrossFilters,
|
||||
filtersInScope,
|
||||
filtersOutOfScope,
|
||||
overflowedCustomizationsInScope = [],
|
||||
renderer,
|
||||
rendererCrossFilter,
|
||||
customizationRenderer,
|
||||
showCollapsePanel,
|
||||
forceRenderOutOfScope,
|
||||
}: FiltersDropdownContentProps) => (
|
||||
@@ -61,6 +76,10 @@ export const FiltersDropdownContent = ({
|
||||
),
|
||||
)}
|
||||
{filtersInScope.map(renderer)}
|
||||
{customizationRenderer &&
|
||||
overflowedCustomizationsInScope.map((item, index) =>
|
||||
customizationRenderer(item, index),
|
||||
)}
|
||||
{showCollapsePanel && filtersOutOfScope.length > 0 && (
|
||||
<FiltersOutOfScopeCollapsible
|
||||
filtersOutOfScope={filtersOutOfScope}
|
||||
|
||||
@@ -195,6 +195,10 @@ function ReportModal({
|
||||
active: true,
|
||||
force_screenshot: false,
|
||||
custom_width: currentReport.custom_width,
|
||||
// A report belongs to either a chart or a dashboard, never both. Explore can
|
||||
// carry dashboard context even for a chart-scoped report, so send only the
|
||||
// entity that matches the creation method; a payload with both `chart` and
|
||||
// `dashboard` is rejected by the backend with a 422 error.
|
||||
...(creationMethod === CreationMethod.Charts
|
||||
? { chart: chart?.id }
|
||||
: { dashboard: dashboardId }),
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
import click
|
||||
@@ -58,10 +59,18 @@ def _load_dataset(
|
||||
if "force" in sig.parameters:
|
||||
params["force"] = force
|
||||
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
loader(**params)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load %s: %s", dataset_name, e)
|
||||
logger.warning(
|
||||
"Failed to load %s after %.2fs: %s",
|
||||
dataset_name,
|
||||
time.perf_counter() - start,
|
||||
e,
|
||||
)
|
||||
else:
|
||||
logger.info("Finished [%s] in %.2fs", dataset_name, time.perf_counter() - start)
|
||||
|
||||
|
||||
def load_examples_run(
|
||||
@@ -70,6 +79,7 @@ def load_examples_run(
|
||||
only_metadata: bool = False,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
run_start = time.perf_counter()
|
||||
if only_metadata:
|
||||
logger.info("Loading examples metadata")
|
||||
else:
|
||||
@@ -94,7 +104,14 @@ def load_examples_run(
|
||||
_load_dataset(loader, loader_name, only_metadata, force)
|
||||
|
||||
# Load examples that are stored as YAML config files
|
||||
configs_start = time.perf_counter()
|
||||
examples.load_examples_from_configs(force, load_test_data)
|
||||
logger.info(
|
||||
"Finished [Examples From Configs] in %.2fs",
|
||||
time.perf_counter() - configs_start,
|
||||
)
|
||||
|
||||
logger.info("load_examples finished in %.2fs", time.perf_counter() - run_start)
|
||||
|
||||
|
||||
@click.command()
|
||||
|
||||
@@ -32,7 +32,7 @@ from superset.commands.exceptions import ImportFailedError
|
||||
from superset.constants import PASSWORD_MASK
|
||||
from superset.databases.ssh_tunnel.models import SSHTunnel
|
||||
from superset.databases.utils import make_url_safe
|
||||
from superset.db_engine_specs.exceptions import SupersetDBAPIConnectionError
|
||||
from superset.db_engine_specs.exceptions import SupersetDBAPIError
|
||||
from superset.exceptions import (
|
||||
OAuth2RedirectError,
|
||||
SupersetSecurityException,
|
||||
@@ -96,6 +96,21 @@ def _refuse_stored_secret_reuse(existing: Database, config: dict[str, Any]) -> N
|
||||
)
|
||||
|
||||
|
||||
def _sync_permissions_best_effort(database: Database) -> None:
|
||||
"""
|
||||
Sync catalog/schema permissions for ``database``, tolerating a transient
|
||||
or OAuth2 failure rather than letting it fail the import.
|
||||
"""
|
||||
try:
|
||||
add_permissions(database)
|
||||
except (SupersetDBAPIError, OAuth2RedirectError) as ex:
|
||||
# ``add_permissions()`` calls ``get_all_catalog_names()`` outside of
|
||||
# its own per-catalog error handling, so any DBAPI error mapped from
|
||||
# that initial catalog discovery -- not just a connection failure --
|
||||
# must be tolerated here too, or it fails the whole import.
|
||||
logger.warning(ex.message)
|
||||
|
||||
|
||||
def import_database( # noqa: C901
|
||||
config: dict[str, Any],
|
||||
overwrite: bool = False,
|
||||
@@ -108,6 +123,15 @@ def import_database( # noqa: C901
|
||||
existing = db.session.query(Database).filter_by(uuid=config["uuid"]).first()
|
||||
if existing:
|
||||
if not overwrite or not can_write:
|
||||
if can_write:
|
||||
# Chart/dataset/saved-query/dashboard bundles that reference
|
||||
# an already-imported database reach this branch; without
|
||||
# this, a schema added to the live connection since the
|
||||
# database was first imported would never get a first-time
|
||||
# grant through this path either. ``add_permissions()`` does
|
||||
# a live, uncached metadata scan, so this can be slow for
|
||||
# cross-catalog-enabled engines -- see its own comment.
|
||||
_sync_permissions_best_effort(existing)
|
||||
return existing
|
||||
config["id"] = existing.id
|
||||
# Stored secrets must not be rebound to a different endpoint: without
|
||||
@@ -184,9 +208,6 @@ def import_database( # noqa: C901
|
||||
recursive=False,
|
||||
)
|
||||
|
||||
try:
|
||||
add_permissions(database)
|
||||
except (SupersetDBAPIConnectionError, OAuth2RedirectError) as ex:
|
||||
logger.warning(ex.message)
|
||||
_sync_permissions_best_effort(database)
|
||||
|
||||
return database
|
||||
|
||||
@@ -22,6 +22,7 @@ from typing import Any, Optional, TypedDict
|
||||
import pandas as pd
|
||||
from flask import current_app
|
||||
from flask_babel import lazy_gettext as _
|
||||
from sqlalchemy import or_
|
||||
from werkzeug.datastructures import FileStorage
|
||||
|
||||
from superset import db
|
||||
@@ -168,12 +169,20 @@ class UploadCommand(BaseCommand):
|
||||
)
|
||||
)
|
||||
|
||||
catalog = self._model.get_default_catalog()
|
||||
|
||||
catalog_filter = (
|
||||
or_(SqlaTable.catalog == catalog, SqlaTable.catalog.is_(None))
|
||||
if catalog is not None
|
||||
else SqlaTable.catalog.is_(None)
|
||||
)
|
||||
sqla_table = (
|
||||
db.session.query(SqlaTable)
|
||||
.filter_by(
|
||||
table_name=self._table_name,
|
||||
schema=self._schema,
|
||||
database_id=self._model_id,
|
||||
.filter(
|
||||
SqlaTable.table_name == self._table_name,
|
||||
SqlaTable.schema == self._schema,
|
||||
SqlaTable.database_id == self._model_id,
|
||||
catalog_filter,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
@@ -206,7 +215,7 @@ class UploadCommand(BaseCommand):
|
||||
)
|
||||
|
||||
if soft_twin := DatasetDAO.find_soft_deleted_logical_duplicate(
|
||||
self._model, Table(self._table_name, self._schema)
|
||||
self._model, Table(self._table_name, self._schema, catalog)
|
||||
):
|
||||
raise DatabaseUploadSoftDeletedDatasetExistsError(str(soft_twin.uuid))
|
||||
|
||||
@@ -217,12 +226,13 @@ class UploadCommand(BaseCommand):
|
||||
table_name=self._table_name,
|
||||
database=self._model,
|
||||
database_id=self._model_id,
|
||||
catalog=catalog,
|
||||
editors=editors,
|
||||
schema=self._schema,
|
||||
# Ensure catalog is set
|
||||
catalog=self._model.get_default_catalog(),
|
||||
)
|
||||
db.session.add(sqla_table)
|
||||
elif sqla_table.catalog is None and catalog is not None:
|
||||
sqla_table.catalog = catalog
|
||||
|
||||
sqla_table.fetch_metadata()
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from superset.constants import PASSWORD_MASK
|
||||
from superset.databases.ssh_tunnel.models import SSHTunnel
|
||||
from superset.databases.utils import make_url_safe
|
||||
from superset.db_engine_specs.base import GenericDBException
|
||||
from superset.exceptions import OAuth2RedirectError
|
||||
from superset.models.core import Database
|
||||
from superset.security.manager import SupersetSecurityManager
|
||||
from superset.utils import json
|
||||
@@ -147,6 +148,29 @@ def ping(engine: Engine) -> bool:
|
||||
return engine.dialect.do_ping(conn)
|
||||
|
||||
|
||||
def _get_all_schema_names_with_retry(
|
||||
database: Database, catalog: str | None
|
||||
) -> set[str]:
|
||||
"""
|
||||
Retry the live schema-listing call once before giving up on a catalog.
|
||||
|
||||
Some catalogs are visible but not listable (eg the ``rdsadmin`` catalog on
|
||||
AWS RDS), but the exception caught by the caller doesn't distinguish that
|
||||
from a one-off transient hiccup (eg schema metadata not yet visible right
|
||||
after it was created). A single retry lets a schema that needs a
|
||||
first-time grant survive a fluke without tolerating a persistently
|
||||
unlistable catalog for any longer than before.
|
||||
"""
|
||||
try:
|
||||
return database.get_all_schema_names(catalog=catalog, cache=False)
|
||||
except OAuth2RedirectError:
|
||||
# Not transient: retrying would just kick off a second, redundant
|
||||
# OAuth2 authorization redirect for the same request.
|
||||
raise
|
||||
except GenericDBException: # pylint: disable=broad-except
|
||||
return database.get_all_schema_names(catalog=catalog, cache=False)
|
||||
|
||||
|
||||
def add_permissions(database: Database) -> None:
|
||||
"""
|
||||
Add DAR for catalogs and schemas.
|
||||
@@ -179,7 +203,8 @@ def add_permissions(database: Database) -> None:
|
||||
|
||||
for catalog in catalogs:
|
||||
try:
|
||||
for schema in database.get_all_schema_names(catalog=catalog, cache=False):
|
||||
schemas = _get_all_schema_names_with_retry(database, catalog)
|
||||
for schema in schemas:
|
||||
security_manager.add_permission_view_menu(
|
||||
"schema_access",
|
||||
security_manager.get_schema_perm(
|
||||
|
||||
@@ -163,6 +163,14 @@ class ImportExamplesCommand(ImportModelsCommand):
|
||||
dataset_info: dict[str, dict[str, Any]] = {}
|
||||
for file_name, config in configs.items():
|
||||
if file_name.startswith("datasets/"):
|
||||
# Some examples ship a dataset config for a table that another
|
||||
# example already defines (same uuid, re-exported under a
|
||||
# different folder). Import each uuid once per run --
|
||||
# reimporting it just repeats the same column/metric sync
|
||||
# against an identical config.
|
||||
if config["uuid"] in dataset_info:
|
||||
continue
|
||||
|
||||
# find the ID of the corresponding database
|
||||
if config["database_uuid"] not in database_ids:
|
||||
raise Exception( # pylint: disable=broad-exception-raised
|
||||
|
||||
@@ -42,6 +42,12 @@ NO_TIME_RANGE = "No filter"
|
||||
|
||||
QUERY_CANCEL_KEY = "cancel_query"
|
||||
QUERY_EARLY_CANCEL_KEY = "early_cancel_query"
|
||||
# Set once execute_sql_statements() has opened a DB connection and asked the
|
||||
# engine spec for a cancel handle, regardless of whether one came back. Lets
|
||||
# cancel_query() tell "hasn't been dispatched to the engine yet" (safe to
|
||||
# fabricate a stop) apart from "this engine just has no cancel support"
|
||||
# (must fail honestly) when no cancel ID is on record.
|
||||
QUERY_DISPATCHED_KEY = "query_dispatched"
|
||||
|
||||
LRU_CACHE_MAX_SIZE = 256
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from superset.queries.filters import QueryFilter
|
||||
from superset.queries.saved_queries.filters import SavedQueryFilter
|
||||
from superset.utils.core import get_user_id
|
||||
from superset.utils.dates import now_as_float
|
||||
from superset.utils.decorators import transaction
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,6 +60,7 @@ class QueryDAO(BaseDAO[Query]):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@transaction()
|
||||
def stop_query(client_id: str) -> None:
|
||||
query = (
|
||||
db.session.query(Query)
|
||||
@@ -81,6 +83,11 @@ class QueryDAO(BaseDAO[Query]):
|
||||
if not sql_lab.cancel_query(query):
|
||||
raise SupersetCancelQueryException("Could not cancel query")
|
||||
|
||||
# cancel_query() may have staged an early-cancel flag on query.extra
|
||||
# without committing it (see its docstring/comments); the
|
||||
# @transaction decorator commits it together with status=STOPPED
|
||||
# below in one transaction, closing the window where another
|
||||
# request could observe the flag set but the status still RUNNING.
|
||||
query.status = QueryStatus.STOPPED
|
||||
query.end_time = now_as_float()
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ import sqlalchemy.dialects
|
||||
from flask import current_app as app
|
||||
from sqlalchemy.engine.default import DefaultDialect
|
||||
from sqlalchemy.exc import NoSuchModuleError
|
||||
from sqlalchemy.sql import compiler as sqla_compiler
|
||||
|
||||
from superset import feature_flag_manager
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
@@ -162,6 +163,16 @@ def get_available_engine_specs() -> dict[type[BaseEngineSpec], set[str]]: # noq
|
||||
continue
|
||||
|
||||
# installed 3rd-party dialects
|
||||
#
|
||||
# `ep.load()` runs arbitrary module-level code in the third-party package.
|
||||
# Some dialects (e.g. sqlalchemy-monetdb) mutate SQLAlchemy's shared,
|
||||
# process-global `compiler.OPERATORS` mapping in place on import instead
|
||||
# of subclassing it, which would otherwise silently change SQL rendering
|
||||
# (e.g. `!=` -> `<>`) for every dialect for the rest of the process, not
|
||||
# just the misbehaving one. Snapshot/restore around each load so a
|
||||
# buggy connector can't leak global compiler state into unrelated
|
||||
# dialects just because it was enumerated here.
|
||||
operators_snapshot = dict(sqla_compiler.OPERATORS)
|
||||
for ep in entry_points(group="sqlalchemy.dialects"):
|
||||
try:
|
||||
dialect = ep.load()
|
||||
@@ -196,6 +207,10 @@ def get_available_engine_specs() -> dict[type[BaseEngineSpec], set[str]]: # noq
|
||||
if isinstance(driver, bytes):
|
||||
driver = driver.decode()
|
||||
drivers[backend].add(driver)
|
||||
finally:
|
||||
if sqla_compiler.OPERATORS != operators_snapshot:
|
||||
sqla_compiler.OPERATORS.clear()
|
||||
sqla_compiler.OPERATORS.update(operators_snapshot)
|
||||
|
||||
dbs_denylist = app.config["DBS_AVAILABLE_DENYLIST"]
|
||||
if not feature_flag_manager.is_feature_enabled("ENABLE_SUPERSET_META_DB"):
|
||||
|
||||
@@ -22,7 +22,7 @@ from datetime import datetime, timezone
|
||||
from typing import Any, cast, TYPE_CHECKING
|
||||
from urllib import parse
|
||||
|
||||
from flask import current_app as app
|
||||
from flask import current_app as app, has_app_context
|
||||
from flask_babel import gettext as __
|
||||
from marshmallow import fields, Schema
|
||||
from marshmallow.validate import Range
|
||||
@@ -366,9 +366,16 @@ try:
|
||||
"*Int128",
|
||||
"string",
|
||||
)
|
||||
# Importing this module happens as a side effect of iterating db_engine_specs
|
||||
# (e.g. from a standalone script or test that never builds a Flask app), so
|
||||
# `current_app` may not be bound to an app context yet -- guard the version
|
||||
# lookup rather than let that crash the import outright.
|
||||
version_string = (
|
||||
app.config.get("VERSION_STRING", "dev") if has_app_context() else "dev"
|
||||
)
|
||||
set_setting(
|
||||
"product_name",
|
||||
f"superset/{app.config.get('VERSION_STRING', 'dev')}",
|
||||
f"superset/{version_string}",
|
||||
)
|
||||
except ImportError: # ClickHouse Connect not installed, do nothing
|
||||
pass
|
||||
|
||||
@@ -44,7 +44,10 @@ class CockroachDbEngineSpec(PostgresEngineSpec):
|
||||
DatabaseCategory.TRADITIONAL_RDBMS,
|
||||
DatabaseCategory.OPEN_SOURCE,
|
||||
],
|
||||
"pypi_packages": ["sqlalchemy-cockroachdb", "psycopg2"],
|
||||
# sqlalchemy-cockroachdb declares no DBAPI dependency of its own (see
|
||||
# the comment on the `cockroachdb` extra in pyproject.toml), so a
|
||||
# plain `cockroachdb://` URL also needs psycopg2 installed to connect.
|
||||
"pypi_packages": ["sqlalchemy-cockroachdb", "psycopg2-binary"],
|
||||
"connection_string": "cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable",
|
||||
"default_port": 26257,
|
||||
"docs_url": "https://github.com/cockroachdb/sqlalchemy-cockroachdb",
|
||||
|
||||
Binary file not shown.
@@ -1,348 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
always_filter_main_dttm: false
|
||||
cache_timeout: null
|
||||
catalog: null
|
||||
columns:
|
||||
- advanced_data_type: null
|
||||
column_name: order_date
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: true
|
||||
python_date_format: null
|
||||
type: TIMESTAMP WITHOUT TIME ZONE
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: price_each
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: DOUBLE PRECISION
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: sales
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: DOUBLE PRECISION
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: address_line1
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: address_line2
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: order_line_number
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: quantity_ordered
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: order_number
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: quarter
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: year
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: month
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: msrp
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: BIGINT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: contact_last_name
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: contact_first_name
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: postal_code
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: customer_name
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: deal_size
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: product_code
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: product_line
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: state
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: status
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: city
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: country
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: phone
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
- advanced_data_type: null
|
||||
column_name: territory
|
||||
description: null
|
||||
expression: null
|
||||
extra: null
|
||||
filterable: true
|
||||
groupby: true
|
||||
is_active: true
|
||||
is_dttm: false
|
||||
python_date_format: null
|
||||
type: TEXT
|
||||
verbose_name: null
|
||||
data_file: cleaned_sales_data.parquet
|
||||
database_uuid: a2dc77af-e654-49bb-b321-40f6b559a1ee
|
||||
default_endpoint: null
|
||||
description: null
|
||||
extra: null
|
||||
fetch_values_predicate: null
|
||||
filter_select_enabled: true
|
||||
folders: null
|
||||
main_dttm_col: order_date
|
||||
metrics:
|
||||
- currency: null
|
||||
d3format: null
|
||||
description: null
|
||||
expression: COUNT(*)
|
||||
extra: null
|
||||
metric_name: count
|
||||
metric_type: count
|
||||
verbose_name: COUNT(*)
|
||||
warning_text: null
|
||||
normalize_columns: false
|
||||
offset: 0
|
||||
params: null
|
||||
schema: null
|
||||
sql: null
|
||||
table_name: cleaned_sales_data
|
||||
template_params: null
|
||||
uuid: e8623bb9-5e00-f531-506a-19607f5f8005
|
||||
version: 1.0.0
|
||||
@@ -150,14 +150,23 @@ def load_parquet_table( # noqa: C901
|
||||
except Exception as e:
|
||||
logger.warning("Could not process column %s: %s", col, e)
|
||||
|
||||
# Write to target database
|
||||
# Write to target database. Scale the row chunksize down for wide
|
||||
# tables so a single batch's bound-parameter count (rows * columns)
|
||||
# stays under stock SQLite's default SQLITE_MAX_VARIABLE_NUMBER of
|
||||
# 32766 -- a flat chunksize=500 on a 328-column table generates
|
||||
# ~164k params per batch, which only some builds raise the limit
|
||||
# for (e.g. Debian's SQLite package). A right-sized batch also
|
||||
# plans faster than an oversized one.
|
||||
num_cols = max(len(pdf.columns), 1)
|
||||
chunksize = max(50, min(500, 30_000 // num_cols))
|
||||
|
||||
with database.get_sqla_engine() as engine:
|
||||
pdf.to_sql(
|
||||
table_name,
|
||||
engine,
|
||||
schema=schema,
|
||||
if_exists="replace",
|
||||
chunksize=500,
|
||||
chunksize=chunksize,
|
||||
method="multi",
|
||||
index=False,
|
||||
)
|
||||
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -689,9 +689,20 @@ def merge_chart_form_data( # noqa: C901
|
||||
if not isinstance(config, GaugeChartConfig):
|
||||
if dataset_rebind:
|
||||
return dict(new_form_data)
|
||||
fields_set = config.model_fields_set
|
||||
if "filters" not in fields_set:
|
||||
preserve_previous_adhoc_filters(new_form_data, existing_form_data)
|
||||
merged = {**existing_form_data, **new_form_data}
|
||||
if getattr(config, "filters", None) == []:
|
||||
merged.pop("adhoc_filters", None)
|
||||
# An explicitly empty collection clears the control rather than
|
||||
# falling through to the inherited value.
|
||||
for config_field, form_data_field in (
|
||||
("filters", "adhoc_filters"),
|
||||
("group_by", "groupby"),
|
||||
("group_by_secondary", "groupby_b"),
|
||||
("sort_by", "order_by_cols"),
|
||||
):
|
||||
if config_field in fields_set and getattr(config, config_field, None) == []:
|
||||
merged.pop(form_data_field, None)
|
||||
return merged
|
||||
|
||||
fields_set = config.model_fields_set
|
||||
@@ -2011,3 +2022,61 @@ def analyze_chart_semantics(viz_type: str | None, config: Any) -> ChartSemantics
|
||||
anomalies=[], # Would need actual data analysis to populate
|
||||
statistical_summary={}, # Would need actual data analysis to populate
|
||||
)
|
||||
|
||||
|
||||
def preserve_previous_adhoc_filters(
|
||||
new_form_data: dict[str, Any], previous_form_data: dict[str, Any]
|
||||
) -> None:
|
||||
"""Preserve saved filters without dropping mapper-generated bindings.
|
||||
|
||||
Saved predicates the caller did not mention survive the update, while the
|
||||
bindings generated for the new config are appended when they are not
|
||||
already represented. A stale temporal binding is dropped when the config
|
||||
rebinds the time filter to a different subject.
|
||||
"""
|
||||
previous_filters = previous_form_data.get("adhoc_filters")
|
||||
if not isinstance(previous_filters, list) or not previous_filters:
|
||||
return
|
||||
|
||||
generated_filters = new_form_data.get("adhoc_filters", [])
|
||||
previous_binding = previous_form_data.get(MCP_DASHBOARD_TIME_FILTER_SUBJECT)
|
||||
new_binding = new_form_data.get(MCP_DASHBOARD_TIME_FILTER_SUBJECT)
|
||||
merged_filters = [
|
||||
filter_
|
||||
for filter_ in previous_filters
|
||||
if not (
|
||||
previous_binding
|
||||
and previous_binding != new_binding
|
||||
and isinstance(filter_, dict)
|
||||
and filter_.get("operator") == FilterOperator.TEMPORAL_RANGE.value
|
||||
and filter_.get("subject") == previous_binding
|
||||
and filter_.get("comparator") == NO_TIME_RANGE
|
||||
)
|
||||
]
|
||||
for generated_filter in generated_filters:
|
||||
if not isinstance(generated_filter, dict):
|
||||
if generated_filter not in merged_filters:
|
||||
merged_filters.append(generated_filter)
|
||||
continue
|
||||
|
||||
# A saved temporal predicate on the same subject wins over the
|
||||
# generated default, so only the comparator-insensitive match is
|
||||
# treated as already represented for TEMPORAL_RANGE.
|
||||
is_same_filter = any(
|
||||
isinstance(previous_filter, dict)
|
||||
and previous_filter.get("clause") == generated_filter.get("clause")
|
||||
and previous_filter.get("expressionType")
|
||||
== generated_filter.get("expressionType")
|
||||
and previous_filter.get("subject") == generated_filter.get("subject")
|
||||
and previous_filter.get("operator") == generated_filter.get("operator")
|
||||
and (
|
||||
generated_filter.get("operator") == FilterOperator.TEMPORAL_RANGE.value
|
||||
or previous_filter.get("comparator")
|
||||
== generated_filter.get("comparator")
|
||||
)
|
||||
for previous_filter in merged_filters
|
||||
)
|
||||
if not is_same_filter:
|
||||
merged_filters.append(generated_filter)
|
||||
|
||||
new_form_data["adhoc_filters"] = merged_filters
|
||||
|
||||
@@ -49,6 +49,7 @@ from superset.mcp_service.chart.schemas import (
|
||||
AccessibilityMetadata,
|
||||
ChartConfig,
|
||||
ColumnRef,
|
||||
GaugeChartConfig,
|
||||
GenerateChartResponse,
|
||||
PerformanceMetadata,
|
||||
TableChartConfig,
|
||||
@@ -211,6 +212,151 @@ def _merge_replacement_config(
|
||||
)
|
||||
|
||||
|
||||
def _valid_dataset_reference(
|
||||
value: Any,
|
||||
columns: set[str],
|
||||
metrics: set[str],
|
||||
*,
|
||||
allow_metric: bool = False,
|
||||
) -> bool:
|
||||
"""Return whether a form_data reference resolves against a dataset."""
|
||||
if not isinstance(value, str):
|
||||
return True
|
||||
normalized = value.casefold()
|
||||
return normalized in columns or (allow_metric and normalized in metrics)
|
||||
|
||||
|
||||
def _inherited_metrics_match_dataset(
|
||||
existing_form_data: dict[str, Any],
|
||||
columns: set[str],
|
||||
metrics: set[str],
|
||||
) -> bool:
|
||||
for metric in existing_form_data.get("metrics") or []:
|
||||
if isinstance(metric, str) and not _valid_dataset_reference(
|
||||
metric, columns, metrics, allow_metric=True
|
||||
):
|
||||
return False
|
||||
if isinstance(metric, dict):
|
||||
column = metric.get("column")
|
||||
if isinstance(column, dict) and not _valid_dataset_reference(
|
||||
column.get("column_name"), columns, metrics
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _inherited_sort_matches_dataset(
|
||||
order_by_cols: Any, columns: set[str], metrics: set[str]
|
||||
) -> bool:
|
||||
for order_by in order_by_cols or []:
|
||||
try:
|
||||
column = json.loads(order_by)[0]
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return False
|
||||
if not _valid_dataset_reference(column, columns, metrics, allow_metric=True):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _inherited_filters_match_dataset(
|
||||
filters: Any, columns: set[str], metrics: set[str]
|
||||
) -> bool:
|
||||
for filter_ in filters or []:
|
||||
if not isinstance(filter_, dict):
|
||||
return False
|
||||
if filter_.get("expressionType") not in (None, "SIMPLE"):
|
||||
return False
|
||||
subject = filter_.get("subject") or filter_.get("col")
|
||||
allow_metric = str(filter_.get("clause", "WHERE")).upper() == "HAVING"
|
||||
if not _valid_dataset_reference(
|
||||
subject, columns, metrics, allow_metric=allow_metric
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
#: form_data keys carrying query roles, mapped to the config field that
|
||||
#: sets them explicitly. An explicit field is the caller's stated intent,
|
||||
#: so it is never treated as inherited state.
|
||||
_INHERITED_QUERY_ROLE_FIELDS = {
|
||||
"groupby": "group_by",
|
||||
"groupby_b": "group_by_secondary",
|
||||
"all_columns": None,
|
||||
"columns": None,
|
||||
"x_axis": None,
|
||||
"granularity_sqla": None,
|
||||
"metrics": None,
|
||||
"order_by_cols": "sort_by",
|
||||
"adhoc_filters": "filters",
|
||||
}
|
||||
|
||||
_INHERITED_COLUMN_LIST_KEYS = frozenset(
|
||||
{"groupby", "groupby_b", "all_columns", "columns"}
|
||||
)
|
||||
_INHERITED_COLUMN_SCALAR_KEYS = frozenset({"x_axis", "granularity_sqla"})
|
||||
|
||||
|
||||
def _inherited_state_invalid_keys(
|
||||
existing_form_data: dict[str, Any],
|
||||
new_form_data: dict[str, Any],
|
||||
parsed_config: ChartConfig,
|
||||
dataset_id: int,
|
||||
) -> set[str]:
|
||||
"""Return inherited query fields that are invalid for a new dataset.
|
||||
|
||||
A dataset rebind only has to discard the state that cannot resolve
|
||||
against the replacement dataset; everything else stays valid and is
|
||||
preserved so the update does not silently reset the chart.
|
||||
"""
|
||||
fields_set = parsed_config.model_fields_set
|
||||
inherited_keys = {
|
||||
key
|
||||
for key, config_field in _INHERITED_QUERY_ROLE_FIELDS.items()
|
||||
if key not in new_form_data
|
||||
and config_field not in fields_set
|
||||
and existing_form_data.get(key)
|
||||
}
|
||||
if not inherited_keys:
|
||||
return set()
|
||||
|
||||
from superset.daos.dataset import DatasetDAO
|
||||
from superset.mcp_service.chart.validation.dataset_validator import (
|
||||
build_dataset_context_from_orm,
|
||||
)
|
||||
|
||||
context = build_dataset_context_from_orm(DatasetDAO.find_by_id(dataset_id))
|
||||
if context is None:
|
||||
# The replacement dataset cannot be inspected, so no inherited
|
||||
# reference can be shown to be safe.
|
||||
return inherited_keys
|
||||
columns = {column["name"].casefold() for column in context.available_columns}
|
||||
metrics = {metric["name"].casefold() for metric in context.available_metrics}
|
||||
|
||||
invalid_keys: set[str] = set()
|
||||
for key in inherited_keys & _INHERITED_COLUMN_LIST_KEYS:
|
||||
values = existing_form_data.get(key)
|
||||
if isinstance(values, list) and not all(
|
||||
_valid_dataset_reference(value, columns, metrics) for value in values
|
||||
):
|
||||
invalid_keys.add(key)
|
||||
for key in inherited_keys & _INHERITED_COLUMN_SCALAR_KEYS:
|
||||
if not _valid_dataset_reference(existing_form_data.get(key), columns, metrics):
|
||||
invalid_keys.add(key)
|
||||
if "metrics" in inherited_keys and not _inherited_metrics_match_dataset(
|
||||
existing_form_data, columns, metrics
|
||||
):
|
||||
invalid_keys.add("metrics")
|
||||
if "order_by_cols" in inherited_keys and not _inherited_sort_matches_dataset(
|
||||
existing_form_data.get("order_by_cols"), columns, metrics
|
||||
):
|
||||
invalid_keys.add("order_by_cols")
|
||||
if "adhoc_filters" in inherited_keys and not _inherited_filters_match_dataset(
|
||||
existing_form_data.get("adhoc_filters"), columns, metrics
|
||||
):
|
||||
invalid_keys.add("adhoc_filters")
|
||||
return invalid_keys
|
||||
|
||||
|
||||
def _build_replacement_form_data(
|
||||
existing_form_data: dict[str, Any],
|
||||
parsed_config: ChartConfig,
|
||||
@@ -222,13 +368,32 @@ def _build_replacement_form_data(
|
||||
parsed_config, dataset_id=effective_dataset_id
|
||||
)
|
||||
new_form_data.pop("_mcp_warnings", None)
|
||||
dataset_rebind = replacement_dataset_id is not None
|
||||
if replacement_dataset_id is not None and not isinstance(
|
||||
parsed_config, GaugeChartConfig
|
||||
):
|
||||
# Drop only the inherited state the replacement dataset cannot
|
||||
# resolve, then merge as a same-dataset update. Gauge keeps the
|
||||
# stricter presentation-only rebind handled downstream.
|
||||
invalid_keys = _inherited_state_invalid_keys(
|
||||
existing_form_data,
|
||||
new_form_data,
|
||||
parsed_config,
|
||||
replacement_dataset_id,
|
||||
)
|
||||
existing_form_data = {
|
||||
key: value
|
||||
for key, value in existing_form_data.items()
|
||||
if key not in invalid_keys
|
||||
}
|
||||
dataset_rebind = False
|
||||
merge_table_column_config(existing_form_data, new_form_data)
|
||||
merge_interactive_pivot_ui_config(existing_form_data, new_form_data)
|
||||
merged = _merge_replacement_config(
|
||||
existing_form_data,
|
||||
new_form_data,
|
||||
parsed_config,
|
||||
dataset_rebind=replacement_dataset_id is not None,
|
||||
dataset_rebind=dataset_rebind,
|
||||
)
|
||||
if replacement_dataset_id is not None:
|
||||
merged["datasource"] = f"{replacement_dataset_id}__table"
|
||||
|
||||
+61
-5
@@ -735,6 +735,66 @@ class BaseSQLStatement(Generic[InternalRepresentation]):
|
||||
return self.format()
|
||||
|
||||
|
||||
_SELECT_TRAILING_CLAUSES: tuple[str, ...] = (
|
||||
"options",
|
||||
"settings",
|
||||
"format",
|
||||
"locks",
|
||||
"offset",
|
||||
"limit",
|
||||
"sort",
|
||||
"cluster",
|
||||
"distribute",
|
||||
"order",
|
||||
"windows",
|
||||
"qualify",
|
||||
"having",
|
||||
"group",
|
||||
"where",
|
||||
"joins",
|
||||
"laterals",
|
||||
"from",
|
||||
"into",
|
||||
"expressions",
|
||||
)
|
||||
|
||||
|
||||
def _get_select_trailing_child(node: exp.Select) -> exp.Expression | None:
|
||||
for clause_name in _SELECT_TRAILING_CLAUSES:
|
||||
val = node.args.get(clause_name)
|
||||
if isinstance(val, list) and val:
|
||||
return _find_last_token_node(val[-1])
|
||||
if isinstance(val, exp.Expression):
|
||||
return _find_last_token_node(val)
|
||||
return None
|
||||
|
||||
|
||||
def _find_last_token_node(node: exp.Expression) -> exp.Expression:
|
||||
"""
|
||||
Find the last token/leaf node in SQL generation order to attach trailing comments.
|
||||
|
||||
Avoids optimizer hints (exp.Hint) and non-trailing subtrees to prevent injecting
|
||||
trailing comments inside optimizer hint blocks (e.g. /*+ SET_VAR(...) */).
|
||||
"""
|
||||
if isinstance(node, exp.Select):
|
||||
if trailing := _get_select_trailing_child(node):
|
||||
return trailing
|
||||
|
||||
children: list[exp.Expression] = []
|
||||
for k, v in node.args.items():
|
||||
if k in ("hint", "comments"):
|
||||
continue
|
||||
if isinstance(v, exp.Expression):
|
||||
children.append(v)
|
||||
elif isinstance(v, list):
|
||||
children.extend(item for item in v if isinstance(item, exp.Expression))
|
||||
|
||||
if children:
|
||||
return _find_last_token_node(children[-1])
|
||||
|
||||
return node
|
||||
|
||||
|
||||
class SQLStatement(BaseSQLStatement[exp.Expression]):
|
||||
"""
|
||||
A SQL statement.
|
||||
@@ -932,11 +992,7 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
|
||||
# statement; move them back to the last token in the last real statement
|
||||
if len(statements) > 1 and isinstance(statements[-1], exp.Semicolon):
|
||||
last_statement = statements.pop()
|
||||
target = statements[-1]
|
||||
for node in statements[-1].walk():
|
||||
if hasattr(node, "comments"): # pragma: no cover
|
||||
target = node
|
||||
|
||||
target = _find_last_token_node(statements[-1])
|
||||
target.comments = target.comments or []
|
||||
target.comments.extend(last_statement.comments)
|
||||
|
||||
|
||||
+177
-5
@@ -39,7 +39,11 @@ from superset import (
|
||||
security_manager,
|
||||
)
|
||||
from superset.common.db_query_status import QueryStatus
|
||||
from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY
|
||||
from superset.constants import (
|
||||
QUERY_CANCEL_KEY,
|
||||
QUERY_DISPATCHED_KEY,
|
||||
QUERY_EARLY_CANCEL_KEY,
|
||||
)
|
||||
from superset.dataframe import df_to_records
|
||||
from superset.db_engine_specs import BaseEngineSpec
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
@@ -99,6 +103,39 @@ def handle_query_error(
|
||||
) -> dict[str, Any]:
|
||||
"""Local method handling error while processing the SQL"""
|
||||
payload = payload or {}
|
||||
|
||||
# A stop request may have already committed STOPPED status while this
|
||||
# exception was being raised/propagated -- this function is the general
|
||||
# catch-all for failures anywhere in execute_sql_statements (connection
|
||||
# setup, cancel-ID acquisition, parsing, or a per-block failure), not
|
||||
# just ones caused by the stop itself. A terminal stop must stay
|
||||
# terminal, so don't let an unrelated error overwrite it with FAILED.
|
||||
#
|
||||
# Deliberately NOT a flush()-then-refresh(query) here, unlike the other
|
||||
# STOPPED-preservation checks in this module: the exception that got us
|
||||
# here may itself have already set query.status (or other attributes)
|
||||
# locally (e.g. SoftTimeLimitExceeded's own handler sets TIMED_OUT
|
||||
# without committing). Flushing first would push that stale local state
|
||||
# to the DB, clobbering a concurrently-committed STOPPED before this
|
||||
# check ever gets to observe it.
|
||||
#
|
||||
# A targeted refresh(attribute_names=["status"]) alone isn't enough:
|
||||
# verified empirically that even though it expires and reloads only the
|
||||
# named attribute (so a dirty `status` itself is correctly discarded
|
||||
# rather than written), the reload's own SELECT still triggers a normal
|
||||
# autoflush of any OTHER dirty attribute on the session first -- e.g. a
|
||||
# pending query.tmp_table_name or query.executed_sql set earlier would
|
||||
# still get written before the status read. no_autoflush suppresses
|
||||
# that: verified it emits only the targeted SELECT, with no UPDATE
|
||||
# beforehand, and leaves other pending attributes exactly as dirty as
|
||||
# they were (to be flushed normally by this function's own commit()
|
||||
# below, once we're past the STOPPED check).
|
||||
with db.session.no_autoflush:
|
||||
db.session.refresh(query, attribute_names=["status"])
|
||||
if query.status == QueryStatus.STOPPED:
|
||||
payload.update({"status": query.status})
|
||||
return payload
|
||||
|
||||
msg = f"{prefix_message} {str(ex)}".strip()
|
||||
query.error_message = msg
|
||||
query.tmp_table_name = None
|
||||
@@ -412,6 +449,21 @@ def execute_sql_statements( # noqa: C901
|
||||
|
||||
query = get_query(query_id=query_id)
|
||||
payload: dict[str, Any] = {"query_id": query_id}
|
||||
|
||||
# A stop request may have landed before this worker even started (e.g.
|
||||
# the request was queued and the user clicked Stop before a worker
|
||||
# picked it up). Honor it here, mirroring the per-block stopped-check
|
||||
# further down, instead of unconditionally overwriting it back to
|
||||
# RUNNING and dispatching the statement anyway.
|
||||
#
|
||||
# Same disclosed, unfixed TOCTOU residual as the other status checks in
|
||||
# this function (see the longer comment above the pre-payload check
|
||||
# further down): a stop committed strictly between this check and the
|
||||
# `query.status = RUNNING` commit a few lines below is still missed.
|
||||
if query.status == QueryStatus.STOPPED:
|
||||
payload.update({"status": query.status})
|
||||
return payload
|
||||
|
||||
database = query.database
|
||||
db_engine_spec = database.db_engine_spec
|
||||
db_engine_spec.patch()
|
||||
@@ -509,9 +561,14 @@ def execute_sql_statements( # noqa: C901
|
||||
cursor = conn.cursor()
|
||||
|
||||
cancel_query_id = db_engine_spec.get_cancel_query_id(cursor, query)
|
||||
# Recorded unconditionally -- even when no cancel ID comes back --
|
||||
# so cancel_query() can tell "hasn't reached the engine yet" (still
|
||||
# safe to fabricate a stop) apart from "this engine has no cancel
|
||||
# support" (must fail honestly) once we get here.
|
||||
query.set_extra_json_key(QUERY_DISPATCHED_KEY, True)
|
||||
if cancel_query_id is not None:
|
||||
query.set_extra_json_key(QUERY_CANCEL_KEY, cancel_query_id)
|
||||
db.session.commit()
|
||||
db.session.commit()
|
||||
|
||||
block_count = len(blocks)
|
||||
for i, block in enumerate(blocks):
|
||||
@@ -564,6 +621,41 @@ def execute_sql_statements( # noqa: C901
|
||||
if parsed_script.has_mutation() or query.select_as_cta:
|
||||
conn.commit()
|
||||
|
||||
# A stop request may have landed after the last per-block check but
|
||||
# before the final statement finished (there's no next iteration to
|
||||
# catch it on for the last block). Check again before building a SUCCESS
|
||||
# payload or writing results to the backend -- both would otherwise
|
||||
# disagree with the row. The results-backend-write-failure branch below
|
||||
# has its own second check for the same reason (a stop landing while
|
||||
# that specific write is in flight).
|
||||
#
|
||||
# KNOWN, DELIBERATELY UNFIXED RESIDUAL: this codebase has no DB-level
|
||||
# locking, so every "check status, then later commit something based on
|
||||
# what was read" pattern in this function -- this one, the
|
||||
# results-backend-write-failure check below, the startup check before
|
||||
# `query.status = RUNNING` is committed a few lines later, and
|
||||
# cancel_query()'s own QUERY_DISPATCHED_KEY read/commit gap (see the
|
||||
# disclosure comment there) -- has the same fundamental TOCTOU window: a
|
||||
# stop committed strictly between the check and the later commit is
|
||||
# still missed. Each check narrows its window as much as reasonably
|
||||
# possible without locking; none of them claim to close it. Closing any
|
||||
# of them for real needs real DB-level row locking (e.g.
|
||||
# SELECT ... FOR UPDATE) or optimistic-concurrency versioning on the
|
||||
# query row, neither of which is meaningfully verifiable against the
|
||||
# sqlite backend this codebase tests against, and is deliberately not
|
||||
# attempted here.
|
||||
#
|
||||
# flush() first: refresh() does NOT autoflush -- without this, any
|
||||
# pending, uncommitted attribute set earlier in this iteration (e.g.
|
||||
# query.executed_sql, set just before execute_query() ran) would be
|
||||
# silently discarded and reloaded back to its previous committed value
|
||||
# instead of surviving to the function's own later commits.
|
||||
db.session.flush()
|
||||
db.session.refresh(query)
|
||||
if query.status == QueryStatus.STOPPED:
|
||||
payload.update({"status": query.status})
|
||||
return payload
|
||||
|
||||
# Success, updating the query entry in database
|
||||
query.rows = result_set.size
|
||||
query.progress = 100
|
||||
@@ -652,6 +744,36 @@ def execute_sql_statements( # noqa: C901
|
||||
# For async queries (not returning results inline), mark as FAILED
|
||||
# because results are inaccessible to the user
|
||||
if not return_results:
|
||||
# A stop request may have landed and committed STOPPED
|
||||
# while this (potentially slow) results-backend write was
|
||||
# in flight. Refresh before marking FAILED -- a terminal
|
||||
# STOPPED must stay terminal, not be overwritten just
|
||||
# because the backend write also failed to complete
|
||||
# around the same time.
|
||||
#
|
||||
# flush() first: refresh() does NOT autoflush -- without
|
||||
# this, the result metadata already set earlier in this
|
||||
# function (rows, progress, extra "columns", select_sql,
|
||||
# end_time) plus the results_key = None set just above
|
||||
# would be silently discarded and reloaded back to their
|
||||
# previous (pre-execution) values instead of surviving to
|
||||
# this branch's own commit below.
|
||||
db.session.flush()
|
||||
db.session.refresh(query)
|
||||
if query.status == QueryStatus.STOPPED:
|
||||
# A fresh, minimal payload -- not `payload.update()`.
|
||||
# By this point `payload` already has the full
|
||||
# SUCCESS shape baked in from earlier (result data, a
|
||||
# nested query["state"] == SUCCESS, and a resultsKey
|
||||
# for a write that just failed), so patching only the
|
||||
# top-level "status" key would return a payload that
|
||||
# simultaneously claims STOPPED while still carrying
|
||||
# SUCCESS data and a resultsKey pointing at nothing
|
||||
# actually stored. Matches the shape the other
|
||||
# STOPPED-preservation return sites in this function
|
||||
# use (a plain {"query_id", "status"} pair).
|
||||
return {"query_id": query_id, "status": query.status}
|
||||
|
||||
query.status = QueryStatus.FAILED
|
||||
query.error_message = (
|
||||
"Failed to store query results in the results backend. "
|
||||
@@ -676,8 +798,24 @@ def execute_sql_statements( # noqa: C901
|
||||
key,
|
||||
)
|
||||
|
||||
# Only set SUCCESS if we didn't already set FAILED above
|
||||
if query.status != QueryStatus.FAILED:
|
||||
# Only set SUCCESS if we didn't already set FAILED above, and don't
|
||||
# clobber a STOPPED status a concurrent stop request may have committed
|
||||
# since the check above -- a terminal stop must stay terminal. This is a
|
||||
# backstop for the DB row specifically (the payload/results-write
|
||||
# consistency check already happened above); it doesn't reopen or
|
||||
# re-narrow the same disclosed race window from that check.
|
||||
#
|
||||
# flush() first: refresh() does NOT autoflush -- without this, every
|
||||
# result field set on the success path above (rows, progress, extra
|
||||
# "columns", select_sql, end_time, results_key) would be silently
|
||||
# discarded and reloaded back to their pre-execution (typically None)
|
||||
# values on EVERY successful query, since nothing before this point
|
||||
# commits them. This was a real regression caught by CI integration
|
||||
# tests across all three DB backends (sqlite/mysql/postgres) that the
|
||||
# unit-test suite driving this fix never exercised.
|
||||
db.session.flush()
|
||||
db.session.refresh(query)
|
||||
if query.status not in (QueryStatus.FAILED, QueryStatus.STOPPED):
|
||||
query.status = QueryStatus.SUCCESS
|
||||
db.session.commit()
|
||||
|
||||
@@ -747,7 +885,41 @@ def cancel_query(query: Query) -> bool:
|
||||
|
||||
cancel_query_id = query.extra.get(QUERY_CANCEL_KEY)
|
||||
if cancel_query_id is None:
|
||||
return False
|
||||
# KNOWN LIMITATION (deliberately not fixed here): this read of
|
||||
# QUERY_DISPATCHED_KEY and execute_sql_statements()'s own commit of
|
||||
# that same flag (see the "Recorded unconditionally" comment where
|
||||
# it's set) are two independent transactions with no lock between
|
||||
# them. A stop request can still land in the narrow window where
|
||||
# this read has already happened -- deciding "not dispatched yet,
|
||||
# safe to fabricate a stop" -- but the worker's dispatch commit
|
||||
# lands immediately after, so the statement still gets sent to the
|
||||
# engine even though the row was just marked STOPPED. Closing this
|
||||
# for real needs DB-level row locking (e.g. SELECT ... FOR UPDATE)
|
||||
# or optimistic-concurrency versioning on the query row; neither is
|
||||
# meaningfully verifiable against the sqlite backend this codebase's
|
||||
# tests run against, so it's out of scope here rather than a
|
||||
# false claim of safety.
|
||||
if query.extra.get(QUERY_DISPATCHED_KEY):
|
||||
# execute_sql_statements() already opened a connection and asked
|
||||
# this engine spec for a cancel handle, and still got nothing --
|
||||
# this engine genuinely has no way to cancel a query once it's
|
||||
# running. That's a real failure, not a race window; report it
|
||||
# honestly rather than fabricating a stop the engine can't back.
|
||||
return False
|
||||
# No cancel handle has been recorded and execution hasn't reached the
|
||||
# engine yet, so "no ID" here can only mean "too early to have one" --
|
||||
# record the same early-cancel intent Trino's own
|
||||
# prepare_cancel_query() records for its harder case (ID only
|
||||
# obtainable after execution starts), so the stopped check at the top
|
||||
# of the statement-block loop honors the request instead of leaving
|
||||
# the query stuck at RUNNING with no avenue to ever stop it.
|
||||
#
|
||||
# Not committed here: the caller (QueryDAO.stop_query) commits this
|
||||
# together with status=STOPPED in one transaction, so another
|
||||
# request can never observe the flag set but the status still
|
||||
# RUNNING.
|
||||
query.set_extra_json_key(QUERY_EARLY_CANCEL_KEY, True)
|
||||
return True
|
||||
|
||||
with query.database.get_sqla_engine(
|
||||
catalog=query.catalog,
|
||||
|
||||
@@ -2487,8 +2487,15 @@ class TestDatabaseApi(SupersetTestCase):
|
||||
}
|
||||
assert response == expected_response
|
||||
|
||||
# Uses a `dialect+driver://` URI (rather than the bare `broken://`
|
||||
# above) to also cover engine-name extraction stripping the driver
|
||||
# suffix. The dialect itself ("broken") must stay one that no
|
||||
# installed extra ever registers a real SQLAlchemy plugin for --
|
||||
# this PR's own testcontainers extras (mssql, oracle, db2, ...)
|
||||
# install real drivers for those dialects, which would make this
|
||||
# URI actually attempt a connection instead of failing to load.
|
||||
data = {
|
||||
"sqlalchemy_uri": "mssql+pymssql://url",
|
||||
"sqlalchemy_uri": "broken+driver://url",
|
||||
"database_name": "examples",
|
||||
"impersonate_user": False,
|
||||
"server_cert": None,
|
||||
@@ -2500,7 +2507,7 @@ class TestDatabaseApi(SupersetTestCase):
|
||||
expected_response = {
|
||||
"errors": [
|
||||
{
|
||||
"message": "Could not load database driver for: mssql",
|
||||
"message": "Could not load database driver for: broken",
|
||||
"error_type": "GENERIC_COMMAND_ERROR",
|
||||
"level": "warning",
|
||||
"extra": {
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Shared import guard for the per-dialect testcontainers modules
|
||||
(tests/testcontainers/db_engine_specs/test_*.py), each of which needs its
|
||||
own optional `testcontainers[...]` driver submodule to even import.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
||||
|
||||
def require_driver(module_name: str) -> None:
|
||||
"""
|
||||
Import `module_name`, a dialect's `testcontainers` driver submodule.
|
||||
|
||||
Most environments treat that driver as optional: a bare local `pytest`
|
||||
run, or another CI job that never installed the `testcontainers` extras,
|
||||
should skip the module rather than fail collection outright.
|
||||
|
||||
The dedicated per-dialect CI job (.github/workflows/testcontainers.yml)
|
||||
sets SUPERSET_TESTCONTAINERS_STRICT, because there the driver is not
|
||||
optional -- that job's matrix installs exactly this one driver for
|
||||
exactly this one module. A broken or missing import there means the job
|
||||
is misconfigured, and should fail loudly instead of silently reporting
|
||||
a misleadingly green, zero-tests-run result.
|
||||
"""
|
||||
if os.environ.get("SUPERSET_TESTCONTAINERS_STRICT"):
|
||||
importlib.import_module(module_name)
|
||||
else:
|
||||
import pytest
|
||||
|
||||
pytest.importorskip(module_name)
|
||||
@@ -0,0 +1,68 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Shared body for the "paginated query returns correct rows in order" test
|
||||
that db_engine_specs.{cockroachdb,crate,db2,mssql,oracle,trino}'s
|
||||
testcontainers suites each run against their own real instance: a plain
|
||||
SQLAlchemy Core LIMIT/OFFSET query, compiled and executed for real. Mocked
|
||||
tests cannot catch a dialect compiling this incorrectly (see
|
||||
apache/superset#42899, where Trino emitted OFFSET before LIMIT) -- only
|
||||
real execution can.
|
||||
|
||||
Each call site keeps its own test function (and dialect-specific docstring)
|
||||
so failures still report against the right module; this only factors out
|
||||
the identical table setup/assert body, via an optional post-insert hook for
|
||||
dialects (CrateDB) that need one, and an optional extra-table-args hook for
|
||||
dialects (ClickHouse) whose CREATE TABLE requires a schema item a plain
|
||||
Column/primary key can't express.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Column, insert, Integer, MetaData, select, Table as SATable
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
|
||||
|
||||
def assert_paginated_query_returns_correct_rows_in_order(
|
||||
engine: Engine,
|
||||
after_insert: Callable[[Connection], None] | None = None,
|
||||
extra_table_args: tuple[Any, ...] = (),
|
||||
) -> None:
|
||||
metadata = MetaData()
|
||||
t = SATable(
|
||||
"pilot_pagination",
|
||||
metadata,
|
||||
# autoincrement=False: a single-column integer primary key otherwise
|
||||
# implicitly becomes AUTO_INCREMENT on MySQL/MariaDB. That column
|
||||
# type treats an explicit 0 as NULL by default (NO_AUTO_VALUE_ON_ZERO
|
||||
# is off), so the id=0 row below would silently get auto-assigned 1,
|
||||
# colliding with the explicit id=1 row in the same batch insert.
|
||||
Column("id", Integer, primary_key=True, autoincrement=False),
|
||||
*extra_table_args,
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(insert(t), [{"id": i} for i in range(10)])
|
||||
if after_insert is not None:
|
||||
after_insert(conn)
|
||||
|
||||
with engine.connect() as conn:
|
||||
stmt = select(t.c.id).order_by(t.c.id).limit(3).offset(4)
|
||||
rows = conn.execute(stmt).fetchall()
|
||||
|
||||
assert [row.id for row in rows] == [4, 5, 6]
|
||||
@@ -0,0 +1,130 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.clickhouse against a real ClickHouse instance, spun
|
||||
up on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
Superset's recommended ClickHouse connector is `clickhouse-connect`
|
||||
(`ClickHouseConnectEngineSpec`, engine "clickhousedb"), which talks HTTP,
|
||||
not `ClickHouseContainer`'s own documented `clickhouse_driver` (a different,
|
||||
native-TCP-protocol package Superset doesn't use at all). The container
|
||||
exposes both the native TCP port (9000) and the HTTP port (8123); this test
|
||||
connects over the HTTP port to match Superset's actual driver.
|
||||
|
||||
Unlike every other dialect in this suite, ClickHouse tables have no real
|
||||
primary key/constraint concept -- CREATE TABLE requires an explicit engine
|
||||
(e.g. MergeTree), or clickhouse-connect's DDL compiler raises a CompileError
|
||||
rather than defaulting to one.
|
||||
|
||||
`superset.db_engine_specs.clickhouse` runs module-level setup code (default
|
||||
type-formatting overrides) that dereferences `current_app.config` whenever
|
||||
clickhouse-connect is installed, so importing it outside a Flask app context
|
||||
raises RuntimeError the first time it's imported in a process.
|
||||
`tests/unit_tests/db_engine_specs/test_clickhouse.py` gets an app context
|
||||
for free from that suite's autouse fixture; this suite has no such fixture,
|
||||
so this test pushes one explicitly around just that one-time import, reusing
|
||||
the real app instance `tests/conftest.py` already builds for the rest of the
|
||||
test run rather than constructing a second one.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.clickhouse")
|
||||
require_driver("clickhouse_connect")
|
||||
|
||||
from clickhouse_connect.cc_sqlalchemy.ddl.tableengine import MergeTree # noqa: E402
|
||||
from testcontainers.community.clickhouse import ClickHouseContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
HTTP_PORT = 8123
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with ClickHouseContainer("clickhouse/clickhouse-server:latest") as container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(HTTP_PORT)
|
||||
yield create_engine(
|
||||
f"clickhousedb://{container.username}:{container.password}"
|
||||
f"@{host}:{port}/{container.dbname}"
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(
|
||||
engine, extra_table_args=(MergeTree(order_by="id"),)
|
||||
)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
ClickHouseConnectEngineSpec.get_columns wraps a real SQLAlchemy
|
||||
Inspector; this exercises that against actual server-reported column
|
||||
metadata rather than a mocked Inspector.
|
||||
"""
|
||||
from tests.integration_tests.test_app import app
|
||||
|
||||
with app.app_context():
|
||||
from superset.db_engine_specs.clickhouse import ClickHouseConnectEngineSpec
|
||||
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
MergeTree(order_by="id"),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = ClickHouseConnectEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = ClickHouseConnectEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,97 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.cockroachdb against a real CockroachDB instance,
|
||||
spun up on demand via testcontainers. Run via
|
||||
.github/workflows/testcontainers.yml -- these exercise real SQL execution
|
||||
and dialect introspection, which mocked unit tests structurally cannot.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.cockroachdb import CockroachDbEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.cockroachdb")
|
||||
|
||||
from testcontainers.community.cockroachdb import CockroachDBContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
# sqlalchemy-cockroachdb registers its dialect under the plain
|
||||
# "cockroachdb" name; the container's own default ("cockroachdb+psycopg2")
|
||||
# matches the abandoned `cockroachdb` package instead (see #43501).
|
||||
with CockroachDBContainer(dialect="cockroachdb") as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
CockroachDbEngineSpec.get_columns wraps a real SQLAlchemy Inspector;
|
||||
this exercises that against actual server-reported column metadata
|
||||
rather than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = CockroachDbEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = CockroachDbEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,103 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.crate against a real CrateDB instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
crate/crate only publishes an amd64 image (no arm64 build), and requires a
|
||||
host CPU supporting the x86-64-v3 instruction set -- QEMU emulation on
|
||||
Apple Silicon cannot satisfy that, so this file cannot run locally on an
|
||||
Apple Silicon machine even with `docker pull --platform linux/amd64`. It
|
||||
runs natively on GitHub Actions' x86_64 runners.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
|
||||
from superset.db_engine_specs.crate import CrateEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.cratedb")
|
||||
|
||||
from testcontainers.community.cratedb import CrateDBContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with CrateDBContainer() as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def _refresh_pilot_pagination(conn: Connection) -> None:
|
||||
# CrateDB is eventually consistent: a row is not guaranteed visible to
|
||||
# subsequent selects immediately after insert.
|
||||
conn.exec_driver_sql("REFRESH TABLE pilot_pagination")
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(
|
||||
engine, after_insert=_refresh_pilot_pagination
|
||||
)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
CrateEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = CrateEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = CrateEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
@@ -0,0 +1,114 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.databend against a real Databend instance, spun up
|
||||
on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
Databend has no dedicated testcontainers module, so this uses a generic
|
||||
DockerContainer against the official `datafuselabs/databend` standalone
|
||||
image. Superset's DatabendEngineSpec defaults to `sslmode=require`
|
||||
(`encryption_parameters`), but the local standalone image has no TLS
|
||||
listener, so this connects with `sslmode=disable` explicitly.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.databend import DatabendEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("databend_sqlalchemy")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
HTTP_PORT = 8000
|
||||
DBNAME = "default"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("datafuselabs/databend")
|
||||
container.with_exposed_ports(HTTP_PORT)
|
||||
# The image's own startup banner documents this exact line as proof its
|
||||
# HTTP query endpoint is bound and ready.
|
||||
container.waiting_for(LogMessageWaitStrategy(f"listened at 0.0.0.0:{HTTP_PORT}"))
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(HTTP_PORT)
|
||||
# "root" with no password is the image's builtin user -- confirmed
|
||||
# directly against a running container, not from the image's own
|
||||
# doc text, which only shows ${USER}/${PASSWORD} placeholders.
|
||||
yield create_engine(f"databend://root:@{host}:{port}/{DBNAME}?sslmode=disable")
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
DatabendEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = DatabendEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = DatabendEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,95 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.db2 against a real IBM Db2 instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
icr.io/db2_community/db2 only publishes amd64/ppc64le/s390x images (no
|
||||
arm64 build), so this cannot run locally on an Apple Silicon machine. It
|
||||
runs natively on GitHub Actions' x86_64 runners. Db2 is also a notably slow
|
||||
starter (a full instance bring-up, not just a process start) -- expect this
|
||||
module alone to take several minutes.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.db2 import Db2EngineSpec
|
||||
from superset.sql.parse import Table
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.db2")
|
||||
|
||||
from testcontainers.community.db2 import Db2Container # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with Db2Container() as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
Db2EngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = Db2EngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = Db2EngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
@@ -0,0 +1,107 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.elasticsearch against a real Elasticsearch instance,
|
||||
spun up on demand via testcontainers. Run via
|
||||
.github/workflows/testcontainers.yml.
|
||||
|
||||
Unlike the SQL-native dialects in this directory, Elasticsearch has no
|
||||
CREATE TABLE / INSERT: indices and documents get created via its REST API
|
||||
(elasticsearch-dbapi's SQLAlchemy dialect is read-focused, translating SQL
|
||||
to the _sql endpoint), matching how Superset actually encounters
|
||||
Elasticsearch in practice -- data arrives via ingestion tooling, not
|
||||
through Superset itself.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.elasticsearch import ElasticSearchEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.elasticsearch")
|
||||
|
||||
from testcontainers.community.elasticsearch import ElasticSearchContainer # noqa: E402
|
||||
|
||||
INDEX = "pilot_pagination"
|
||||
|
||||
|
||||
def _index_document(
|
||||
base_url: str, index: str, doc_id: int, body: dict[str, int]
|
||||
) -> None:
|
||||
response = requests.put(f"{base_url}/{index}/_doc/{doc_id}", json=body, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def _refresh(base_url: str, index: str) -> None:
|
||||
response = requests.post(f"{base_url}/{index}/_refresh", timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with ElasticSearchContainer("elasticsearch:8.11.0") as container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(container.port)
|
||||
base_url = f"http://{host}:{port}"
|
||||
|
||||
for i in range(10):
|
||||
_index_document(base_url, INDEX, i, {"id": i})
|
||||
_refresh(base_url, INDEX)
|
||||
|
||||
yield create_engine(f"elasticsearch+http://{host}:{port}/")
|
||||
|
||||
|
||||
def test_ordered_limited_query_returns_correct_rows(engine: Engine) -> None:
|
||||
"""
|
||||
A plain LIMIT query, compiled and executed against a real instance.
|
||||
Mocked tests cannot catch a dialect compiling this incorrectly (see
|
||||
apache/superset#42899, where Trino emitted OFFSET before LIMIT) -- only
|
||||
real execution can. No OFFSET here: Elasticsearch's SQL layer genuinely
|
||||
doesn't support it (a protocol limitation, not a bug -- confirmed
|
||||
against a real instance, which raises a parsing_exception on OFFSET).
|
||||
ElasticSearchEngineSpec.supports_offset = False documents this already.
|
||||
"""
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(f"SELECT id FROM {INDEX} ORDER BY id LIMIT 3") # noqa: S608
|
||||
).fetchall()
|
||||
|
||||
assert [row.id for row in rows] == [0, 1, 2]
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
ElasticSearchEngineSpec.get_columns wraps a real SQLAlchemy Inspector;
|
||||
this exercises that against actual server-reported field mappings
|
||||
rather than a mocked Inspector.
|
||||
"""
|
||||
inspector = inspect(engine)
|
||||
columns = ElasticSearchEngineSpec.get_columns(inspector, Table(INDEX))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert "id" in by_name
|
||||
spec = ElasticSearchEngineSpec.get_column_spec(str(by_name["id"]["type"]))
|
||||
assert spec is not None
|
||||
@@ -0,0 +1,146 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.firebird against a real Firebird instance, spun up
|
||||
on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
Firebird references a database *file* rather than a server-managed named
|
||||
database -- the connection URI is `firebird://user:pass@host:port/<path>`,
|
||||
where <path> is the path to a .fdb file on the server. This uses the
|
||||
well-known `jacobalberty/firebird` image, which creates that file (per
|
||||
FIREBIRD_DATABASE) under /firebird/data on first boot.
|
||||
|
||||
FirebirdEngineSpec sets `limit_method = LimitMethod.FETCH_MANY` with a
|
||||
comment claiming Firebird "uses FIRST to limit" -- stale relative to the
|
||||
modern sqlalchemy-firebird driver, which compiles real ROWS-based
|
||||
pagination (confirmed via an offline dialect compile: `SELECT ... ROWS
|
||||
4 + 1 TO 4 + 3`, correctly ordered, not a Trino-style bug). That staleness
|
||||
affects what Superset's own query layer emits, not what this suite's
|
||||
direct dialect-compilation check exercises.
|
||||
|
||||
Could not be verified against a real running instance in this
|
||||
environment: `firebird-driver` is a pure-Python ctypes wrapper (its wheel
|
||||
is `py3-none-any`, confirmed by downloading it directly) that dynamically
|
||||
loads the native Firebird client library (`libfbclient`) from the host at
|
||||
import time -- it doesn't bundle that library itself. This machine has no
|
||||
Homebrew formula or straightforward install path for it. The container
|
||||
itself was confirmed to start and pass its own healthcheck; CI installs
|
||||
the `libfbclient2` system package separately (see
|
||||
.github/workflows/testcontainers.yml) for the actual client-library
|
||||
dependency this driver needs.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.firebird import FirebirdEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("firebird.driver")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import HealthcheckWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
PORT = 3050
|
||||
PASSWORD = "masterkey" # noqa: S105 -- fixed test-fixture password, not a secret
|
||||
DB_FILE = "test.fdb"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("jacobalberty/firebird")
|
||||
container.with_exposed_ports(PORT)
|
||||
container.with_env("ISC_PASSWORD", PASSWORD)
|
||||
container.with_env("FIREBIRD_DATABASE", DB_FILE)
|
||||
# The image logs nothing beyond a single startup banner line and never
|
||||
# prints a distinct "ready" message -- it ships its own Docker
|
||||
# HEALTHCHECK instead, confirmed via `docker ps` reporting (healthy).
|
||||
container.waiting_for(HealthcheckWaitStrategy())
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(PORT)
|
||||
eng = create_engine(
|
||||
f"firebird://sysdba:{PASSWORD}@{host}:{port}//firebird/data/{DB_FILE}"
|
||||
)
|
||||
yield eng
|
||||
# firebird-driver registers its own atexit handler that talks to
|
||||
# the Firebird subsystem to shut it down cleanly. Without disposing
|
||||
# here first, that handler fires at interpreter exit against a
|
||||
# server the container has *already* torn down -- confirmed on
|
||||
# real CI as a segfault (exit code 139) after both tests had
|
||||
# already passed. Disposing while the server is still up lets the
|
||||
# driver close out normally, so the later atexit call has nothing
|
||||
# left to talk to.
|
||||
eng.dispose()
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
FirebirdEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = FirebirdEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = FirebirdEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,117 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.mariadb against a real MariaDB instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
MariaDB is a MySQL fork implementing the same wire protocol: connects via
|
||||
the plain "mysql" dialect with mysqlclient, same as vanilla MySQL, just
|
||||
pointed at the mariadb image instead of mysql:latest.
|
||||
|
||||
Could not be verified locally in this environment: mysqlclient (MySQLdb)
|
||||
has a pre-existing, unrelated native-library linking issue against this
|
||||
machine's Homebrew-installed libmysqlclient. CI installs it via apt on
|
||||
Linux, where this does not occur.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.mariadb import MariaDBEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.mysql")
|
||||
|
||||
from testcontainers.community.mysql import MySqlContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with MySqlContainer("mariadb:11") as container:
|
||||
# get_connection_url() has no host override and defaults to
|
||||
# get_container_host_ip(), which is the literal string "localhost"
|
||||
# on native Linux Docker (e.g. GitHub Actions runners). MySQLdb
|
||||
# (mysqlclient) treats a "localhost" host specially and attempts a
|
||||
# Unix socket connection instead of TCP, which fails since there's
|
||||
# no local MySQL socket -- the container is reached over the
|
||||
# network. Only rewrite that specific local case to 127.0.0.1; a
|
||||
# remote Docker daemon reports its own real host/IP here, which
|
||||
# must be preserved so the suite can still reach it.
|
||||
host = container.get_container_host_ip()
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
port = container.get_exposed_port(container.port)
|
||||
yield create_engine(
|
||||
f"mysql://{container.username}:{container.password}"
|
||||
f"@{host}:{port}/{container.dbname}"
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
MariaDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = MariaDBEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = MariaDBEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,127 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.monetdb against a real MonetDB instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
monetdb/monetdb publishes an amd64-only image, so this needs Rosetta/QEMU
|
||||
emulation on Apple Silicon -- unlike CrateDB's x86-64-v3 CPU requirement,
|
||||
this one actually runs fine under emulation (verified locally). No native
|
||||
testcontainers module exists for MonetDB, so this uses a generic
|
||||
DockerContainer with the documented MDB_* environment variables and waits
|
||||
for the daemon's own startup log line.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.monetdb import MonetDbEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("sqlalchemy_monetdb")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import ( # noqa: E402
|
||||
CompositeWaitStrategy,
|
||||
LogMessageWaitStrategy,
|
||||
PortWaitStrategy,
|
||||
)
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
PORT = 50000
|
||||
PASSWORD = "monetdb" # noqa: S105 -- fixed test-fixture password, not a secret
|
||||
DBNAME = "test"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("monetdb/monetdb:latest")
|
||||
container.with_exposed_ports(PORT)
|
||||
container.with_env("MDB_DB_ADMIN_PASS", PASSWORD)
|
||||
container.with_env("MDB_CREATE_DBS", DBNAME)
|
||||
# The "Starting MonetDB daemon" log line is emitted before the image
|
||||
# actually runs `monetdbd start -n`, so it alone isn't proof the server
|
||||
# is accepting connections yet. Follow it with a port-connect check,
|
||||
# which only succeeds once monetdbd is really listening.
|
||||
container.waiting_for(
|
||||
CompositeWaitStrategy(
|
||||
LogMessageWaitStrategy(re.compile("Starting MonetDB daemon")),
|
||||
PortWaitStrategy(PORT),
|
||||
)
|
||||
)
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(PORT)
|
||||
yield create_engine(f"monetdb://monetdb:{PASSWORD}@{host}:{port}/{DBNAME}")
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
MonetDbEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = MonetDbEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = MonetDbEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,141 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.mongodb against a real MongoDB instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
MongoDB is schemaless, and Superset talks to it via `pymongosql`, a
|
||||
SQL-to-MongoDB translation layer (dialect requires a `?mode=superset` query
|
||||
param -- not part of testcontainers' own MongoDbContainer.get_connection_url()).
|
||||
Documents get inserted via the native pymongo driver, not SQL INSERT,
|
||||
matching how Superset actually encounters MongoDB in practice and avoiding
|
||||
any assumption about pymongosql's own INSERT/DDL support.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
select,
|
||||
table,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.mongodb import MongoDBEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.mongodb")
|
||||
require_driver("pymongosql")
|
||||
|
||||
from testcontainers.community.mongodb import MongoDbContainer # noqa: E402
|
||||
|
||||
COLLECTION = "pilot_pagination"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with MongoDbContainer("mongo:7.0.7") as container:
|
||||
client = container.get_connection_client()
|
||||
client[container.dbname][COLLECTION].insert_many([{"id": i} for i in range(10)])
|
||||
# MongoDbContainer.get_connection_url() has no database path segment
|
||||
# or query string at all (it only builds user:pass@host:port), so
|
||||
# naively appending "&mode=superset" glues it straight onto the port
|
||||
# number instead of starting a query string. Build the full URL
|
||||
# ourselves instead of relying on string concatenation.
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(container.port)
|
||||
# authSource=admin is required: MongoDbContainer creates its root
|
||||
# user via MONGO_INITDB_ROOT_USERNAME, which lives in the `admin`
|
||||
# database, not in `dbname` -- without it, auth fails against
|
||||
# whatever database is in the URL path.
|
||||
yield create_engine(
|
||||
f"mongodb://{container.username}:{container.password}@{host}:{port}"
|
||||
f"/{container.dbname}?mode=superset&authSource=admin"
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed
|
||||
against a real instance. Mocked tests cannot catch a dialect compiling
|
||||
this incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can. Unlike Elasticsearch's SQL
|
||||
layer (which has no OFFSET support at all), pymongosql maps OFFSET to
|
||||
MongoDB's native `skip`, so this dialect supports it.
|
||||
|
||||
Compiled with `literal_binds=True`, matching how Superset actually
|
||||
issues chart/SQL Lab queries (see `models/helpers.py`'s
|
||||
`get_query_str_extended`): pymongosql's SQL-to-Mongo AST parser reads
|
||||
LIMIT/OFFSET straight off the compiled SQL text ahead of parameter
|
||||
substitution, so a bound `LIMIT ?`/`OFFSET ?` placeholder is rejected
|
||||
("invalid literal for int() with base 10: '?'") and the clause is
|
||||
silently dropped -- unlike its WHERE-clause parameter handling, which
|
||||
does substitute correctly. Literal binds sidestep that and exercise
|
||||
the dialect's actual LIMIT/OFFSET compilation, per this test's intent.
|
||||
|
||||
Uses a bare `column("id")`/`table(...)` pair rather than a full
|
||||
`Table`-bound column: SQLAlchemy always qualifies a Table-bound column
|
||||
reference as `pilot_pagination.id` once there's a FROM clause, and
|
||||
pymongosql's projection builder takes that qualified text completely
|
||||
literally as a MongoDB field path -- `{"pilot_pagination.id": 1}` reads
|
||||
a *nested* field under a top-level `pilot_pagination` key, which
|
||||
doesn't exist on these flat documents, silently projecting None instead
|
||||
of raising. An unbound column compiles unqualified ("id"), which
|
||||
resolves correctly, while still exercising the dialect's own
|
||||
LIMIT/OFFSET compilation via a real Core `select()`.
|
||||
"""
|
||||
id_col = column("id")
|
||||
stmt = (
|
||||
select(id_col)
|
||||
.select_from(table(COLLECTION))
|
||||
.order_by(id_col)
|
||||
.limit(3)
|
||||
.offset(4)
|
||||
)
|
||||
compiled = stmt.compile(engine, compile_kwargs={"literal_binds": True})
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(text(str(compiled))).fetchall()
|
||||
|
||||
assert [row.id for row in rows] == [4, 5, 6]
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
MongoDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector, which
|
||||
pymongosql implements by sampling real documents to infer column types
|
||||
-- this exercises that against an actual running instance rather than
|
||||
a mocked Inspector.
|
||||
"""
|
||||
inspector = inspect(engine)
|
||||
columns = MongoDBEngineSpec.get_columns(inspector, Table(COLLECTION))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert "id" in by_name
|
||||
spec = MongoDBEngineSpec.get_column_spec(str(by_name["id"]["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,93 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.mssql against a real SQL Server instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
mcr.microsoft.com/mssql/server only publishes an amd64 image (SQL Server on
|
||||
Linux has no ARM build), so this cannot run locally on an Apple Silicon
|
||||
machine. It runs natively on GitHub Actions' x86_64 runners.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.mssql import MssqlEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.mssql")
|
||||
|
||||
from testcontainers.community.mssql import SqlServerContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with SqlServerContainer() as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
MssqlEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = MssqlEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = MssqlEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
@@ -0,0 +1,117 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.mysql against a real MySQL instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
Plain MySQL itself was never covered by this suite: MariaDB and StarRocks
|
||||
both reuse `MySQLEngineSpec`'s plain "mysql" dialect via mysqlclient, but
|
||||
neither stands in for vanilla MySQL server's own dialect quirks.
|
||||
|
||||
Could not be verified locally in this environment: mysqlclient (MySQLdb)
|
||||
has a pre-existing, unrelated native-library linking issue against this
|
||||
machine's Homebrew-installed libmysqlclient. CI installs it via apt on
|
||||
Linux, where this does not occur.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.mysql")
|
||||
|
||||
from testcontainers.community.mysql import MySqlContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with MySqlContainer("mysql:8.0") as container:
|
||||
# get_connection_url() has no host override and defaults to
|
||||
# get_container_host_ip(), which is the literal string "localhost"
|
||||
# on native Linux Docker (e.g. GitHub Actions runners). MySQLdb
|
||||
# (mysqlclient) treats a "localhost" host specially and attempts a
|
||||
# Unix socket connection instead of TCP, which fails since there's
|
||||
# no local MySQL socket -- the container is reached over the
|
||||
# network. Only rewrite that specific local case to 127.0.0.1; a
|
||||
# remote Docker daemon reports its own real host/IP here, which
|
||||
# must be preserved so the suite can still reach it.
|
||||
host = container.get_container_host_ip()
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
port = container.get_exposed_port(container.port)
|
||||
yield create_engine(
|
||||
f"mysql://{container.username}:{container.password}"
|
||||
f"@{host}:{port}/{container.dbname}"
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
MySQLEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = MySQLEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = MySQLEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,221 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.oceanbase against a real OceanBase instance, spun up
|
||||
on demand via testcontainers. Run via .github/workflows/testcontainers.yml,
|
||||
on the nightly cron / manual dispatch only (see `nightly_only: true` on
|
||||
this dialect's matrix entry) -- OceanBase bootstraps a distributed-style
|
||||
cluster even in single-node MODE=MINI, a substantially heavier first-boot
|
||||
than a single-process database, not a good fit for every PR's CI budget.
|
||||
|
||||
OceanBaseEngineSpec extends MySQLEngineSpec and its dialect
|
||||
(oceanbase_py.sqlalchemy.dialect.OceanBaseDialect) extends
|
||||
MySQLDialect_mysqldb directly with no custom DDL or LIMIT/OFFSET compiler,
|
||||
so this follows the same mysqlclient-based pattern as MariaDB/MySQL/
|
||||
StarRocks in this suite -- including the same "localhost" -> "127.0.0.1"
|
||||
fix MySQLdb needs on native Linux Docker.
|
||||
|
||||
Could not be verified locally in this environment: mysqlclient (MySQLdb)
|
||||
has a pre-existing, unrelated native-library linking issue against this
|
||||
machine's Homebrew-installed libmysqlclient, and this dialect wasn't
|
||||
pulled/run locally at all given its heavier resource footprint -- CI-only
|
||||
verification, matching the nightly_only gating.
|
||||
|
||||
oceanbase_py.sqlalchemy.dialect.OceanBaseDialect has real bugs, all
|
||||
confirmed on real CI, in both has_table() (called by create_all()'s
|
||||
default checkfirst=True) and get_columns() (called by
|
||||
OceanBaseEngineSpec.get_columns(), which this suite's second test needs
|
||||
to actually exercise):
|
||||
1. Both pass a raw string straight to Connection.execute() (e.g.
|
||||
`connection.execute(f"DESCRIBE {full_name}")`), which SQLAlchemy 2.0
|
||||
rejects outright (ObjectNotExecutableError). Every *other* raw-SQL
|
||||
method in the same dialect module correctly uses
|
||||
`connection.exec_driver_sql(...)` instead.
|
||||
2. has_table() never catches the error DESCRIBE raises for a table that
|
||||
doesn't exist (1146) -- so even with (1) fixed, it can only ever
|
||||
return True, raising instead of returning False for exactly the case
|
||||
checkfirst exists to handle.
|
||||
This test monkeypatches both methods to do what the rest of the dialect's
|
||||
raw-SQL methods already do, plus has_table()'s missing not-found
|
||||
handling, rather than working around any of this from the test side.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Connection, Engine, URL
|
||||
from sqlalchemy.exc import NoSuchTableError, ProgrammingError
|
||||
|
||||
from superset.db_engine_specs.oceanbase import OceanBaseEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("oceanbase_py")
|
||||
|
||||
from oceanbase_py.sqlalchemy import datatype # noqa: E402
|
||||
from oceanbase_py.sqlalchemy.dialect import OceanBaseDialect # noqa: E402
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
def _has_table(
|
||||
self: OceanBaseDialect,
|
||||
connection: Connection,
|
||||
table_name: str,
|
||||
schema: str | None = None,
|
||||
**kw: object,
|
||||
) -> bool:
|
||||
if schema is None:
|
||||
schema = self.default_schema_name
|
||||
quote = self.identifier_preparer.quote_identifier
|
||||
full_name = quote(table_name)
|
||||
if schema:
|
||||
full_name = f"{quote(schema)}.{full_name}"
|
||||
try:
|
||||
res = connection.exec_driver_sql(f"DESCRIBE {full_name}")
|
||||
except ProgrammingError:
|
||||
# The original never catches this at all -- DESCRIBE on a
|
||||
# nonexistent table raises 1146 ("table doesn't exist") rather
|
||||
# than returning an empty result set, so the unpatched method can
|
||||
# only ever return True, and raises instead of returning False for
|
||||
# exactly the case create_all()'s checkfirst exists to handle
|
||||
# (confirmed on real CI).
|
||||
return False
|
||||
return res.first() is not None
|
||||
|
||||
|
||||
def _get_columns(
|
||||
self: OceanBaseDialect,
|
||||
connection: Connection,
|
||||
table_name: str,
|
||||
schema: str | None = None,
|
||||
**kw: object,
|
||||
) -> list[dict[str, Any]]:
|
||||
# Same connection.execute(raw string) anti-pattern as has_table(),
|
||||
# confirmed on real CI as the same ObjectNotExecutableError -- this is
|
||||
# the actual column-introspection call OceanBaseEngineSpec.get_columns
|
||||
# (and this test) needs, so it gets the same exec_driver_sql fix.
|
||||
if not self.has_table(connection, table_name, schema):
|
||||
raise NoSuchTableError(f"schema={schema}, table={table_name}")
|
||||
schema = schema or self._get_default_schema_name(connection)
|
||||
|
||||
quote = self.identifier_preparer.quote_identifier
|
||||
full_name = quote(table_name)
|
||||
if schema:
|
||||
full_name = f"{quote(schema)}.{full_name}"
|
||||
|
||||
res = connection.exec_driver_sql(f"SHOW COLUMNS FROM {full_name}")
|
||||
return [
|
||||
{
|
||||
"name": record.Field,
|
||||
"type": datatype.parse_sql_type(record.Type),
|
||||
"nullable": record.Null == "YES",
|
||||
"default": record.Default,
|
||||
}
|
||||
for record in res
|
||||
]
|
||||
|
||||
|
||||
OceanBaseDialect.has_table = _has_table
|
||||
OceanBaseDialect.get_columns = _get_columns
|
||||
|
||||
PORT = 2881
|
||||
PASSWORD = "pilot" # noqa: S105 -- fixed test-fixture password, not a secret
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("oceanbase/oceanbase-ce")
|
||||
container.with_exposed_ports(PORT)
|
||||
container.with_env("MODE", "MINI")
|
||||
container.with_env("OB_TENANT_PASSWORD", PASSWORD)
|
||||
container.waiting_for(LogMessageWaitStrategy("boot success!"))
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
port = container.get_exposed_port(PORT)
|
||||
# OceanBase usernames for a MySQL-mode tenant use "user@tenant"
|
||||
# (e.g. "root@test"), a literal "@" that URL.create() percent-encodes
|
||||
# correctly -- an f-string would produce a second "@" that breaks
|
||||
# the URL's own host/user boundary parsing.
|
||||
yield create_engine(
|
||||
URL.create(
|
||||
"oceanbase",
|
||||
username="root@test",
|
||||
password=PASSWORD,
|
||||
host=host,
|
||||
port=int(port),
|
||||
database="test",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
OceanBaseEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = OceanBaseEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = OceanBaseEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,95 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.oracle against a real Oracle instance (gvenzl/oracle-free),
|
||||
spun up on demand via testcontainers. Run via
|
||||
.github/workflows/testcontainers.yml.
|
||||
|
||||
gvenzl/oracle-free ships with its datafiles pre-baked into the image, so
|
||||
once the (large-ish, ~1GB) image is pulled, container startup is fast --
|
||||
under 15s measured locally. Almost all the wall-clock cost here is the
|
||||
image pull itself, same as any other dialect's container.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.oracle import OracleEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.oracle")
|
||||
|
||||
from testcontainers.community.oracle import OracleDbContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with OracleDbContainer() as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
OracleEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = OracleEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = OracleEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
@@ -0,0 +1,99 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.postgres against a real PostgreSQL instance, spun up
|
||||
on demand via testcontainers. Run via .github/workflows/testcontainers.yml
|
||||
-- these exercise real SQL execution and dialect introspection, which
|
||||
mocked unit tests structurally cannot.
|
||||
|
||||
Plain Postgres itself was never covered by this suite: CockroachDB,
|
||||
TimescaleDB and YugabyteDB all speak the Postgres wire protocol and already
|
||||
exercise `PostgresContainer`/the "postgresql" dialect, but none of them
|
||||
stand in for vanilla PostgreSQL's own dialect quirks.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.postgres")
|
||||
|
||||
from testcontainers.community.postgres import PostgresContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with PostgresContainer("postgres:17-alpine") as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
PostgresEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = PostgresEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = PostgresEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,127 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.risingwave against a real RisingWave instance, spun
|
||||
up on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
RisingWave speaks the Postgres wire protocol, but doesn't run the real
|
||||
Postgres server binary or its POSTGRES_PASSWORD-style bootstrap env vars,
|
||||
so this can't reuse `PostgresContainer` the way TimescaleDB/YugabyteDB do
|
||||
-- it needs a generic DockerContainer against the official
|
||||
`risingwavelabs/risingwave` single-binary playground image instead.
|
||||
`RisingWaveDbEngineSpec` extends `PostgresEngineSpec`, and
|
||||
`sqlalchemy-risingwave`'s dialect is a genuine subclass of SQLAlchemy's own
|
||||
Postgres dialect (via psycopg2), so DDL/pagination compile with standard
|
||||
Postgres semantics -- no ClickHouse-style mandatory table option needed.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
|
||||
from superset.db_engine_specs.risingwave import RisingWaveDbEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("sqlalchemy_risingwave")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
PORT = 4566
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("risingwavelabs/risingwave")
|
||||
container.with_exposed_ports(PORT)
|
||||
container.with_command("playground")
|
||||
# The actual startup banner reads "RisingWave standalone mode is
|
||||
# ready." -- confirmed against a real container's logs.
|
||||
container.waiting_for(
|
||||
LogMessageWaitStrategy(re.compile("RisingWave standalone mode is ready"))
|
||||
)
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(PORT)
|
||||
yield create_engine(f"risingwave://root@{host}:{port}/dev")
|
||||
|
||||
|
||||
def _flush(conn: Connection) -> None:
|
||||
# RisingWave's storage engine checkpoints asynchronously: without an
|
||||
# explicit FLUSH, a SELECT immediately after INSERT can see zero rows
|
||||
# -- confirmed against a real instance (a bare INSERT commits fine, but
|
||||
# the data isn't visible to a subsequent query until flushed).
|
||||
conn.execute(text("FLUSH"))
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine, after_insert=_flush)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
RisingWaveDbEngineSpec.get_columns wraps a real SQLAlchemy Inspector;
|
||||
this exercises that against actual server-reported column metadata
|
||||
rather than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = RisingWaveDbEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = RisingWaveDbEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,156 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.starrocks against a real StarRocks instance, spun up
|
||||
on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
StarRocks has no dedicated testcontainers module, so this uses a generic
|
||||
DockerContainer against the official `starrocks/allin1-ubuntu` image, which
|
||||
brings up both the FE (query frontend, MySQL wire protocol on port 9030)
|
||||
and BE (execution backend) in a single container -- a heavier bring-up than
|
||||
a single-process database. `root` has no password by default and no
|
||||
database exists yet, so the fixture creates one itself before yielding an
|
||||
engine pointed at it. The FE's query port accepts connections, and can even
|
||||
run metadata statements like CREATE DATABASE, before the BE has registered
|
||||
with it -- an actual CREATE TABLE/INSERT then fails with "Backend node not
|
||||
found" -- so the fixture retries a real create-table-and-insert probe
|
||||
against a throwaway table rather than trusting the open port or a bare
|
||||
CREATE DATABASE as a readiness signal.
|
||||
|
||||
Not verified locally in this environment: the `allin1-ubuntu` image is
|
||||
multiple GB and was skipped here to keep local Docker resource usage low,
|
||||
per session guidance to lean on CI (which has no such constraint) for
|
||||
dialects with unusually heavy images.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.starrocks import StarRocksEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("starrocks")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import PortWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
QUERY_PORT = 9030
|
||||
DBNAME = "pilot"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("starrocks/allin1-ubuntu")
|
||||
container.with_exposed_ports(QUERY_PORT)
|
||||
container.waiting_for(PortWaitStrategy(QUERY_PORT))
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(QUERY_PORT)
|
||||
bootstrap_engine = create_engine(
|
||||
f"starrocks://root:@{host}:{port}/default_catalog.information_schema"
|
||||
)
|
||||
|
||||
# The FE's query port accepts connections, and can even run metadata
|
||||
# statements like CREATE DATABASE, before any BE (execution backend)
|
||||
# has registered with it -- an actual table create/insert then fails
|
||||
# with "Backend node not found". Probe with the real operations the
|
||||
# tests below need, in a throwaway table, so readiness is confirmed
|
||||
# for what actually matters rather than just the FE's own port.
|
||||
last_error: Exception | None = None
|
||||
for _ in range(60):
|
||||
try:
|
||||
with bootstrap_engine.begin() as conn:
|
||||
conn.execute(text(f"CREATE DATABASE IF NOT EXISTS {DBNAME}"))
|
||||
probe_engine = create_engine(
|
||||
f"starrocks://root:@{host}:{port}/default_catalog.{DBNAME}"
|
||||
)
|
||||
with probe_engine.begin() as conn:
|
||||
conn.execute(
|
||||
text("CREATE TABLE IF NOT EXISTS pilot_ready (id INT)")
|
||||
)
|
||||
conn.execute(text("INSERT INTO pilot_ready VALUES (1)"))
|
||||
conn.execute(text("DROP TABLE pilot_ready"))
|
||||
break
|
||||
except Exception as ex: # noqa: BLE001 -- retry on any not-ready-yet error
|
||||
last_error = ex
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"StarRocks FE/BE never became ready to create and use a table"
|
||||
) from last_error
|
||||
|
||||
yield create_engine(f"starrocks://root:@{host}:{port}/default_catalog.{DBNAME}")
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
StarRocksEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = StarRocksEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = StarRocksEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,98 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.timescaledb against a real TimescaleDB instance,
|
||||
spun up on demand via testcontainers. Run via
|
||||
.github/workflows/testcontainers.yml -- these exercise real SQL execution
|
||||
and dialect introspection, which mocked unit tests structurally cannot.
|
||||
|
||||
TimescaleDB is a genuine Postgres extension, not a fork: connects via the
|
||||
plain "postgresql" dialect with psycopg2, same as vanilla Postgres, just
|
||||
pointed at the timescale/timescaledb image instead of postgres:latest.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.timescaledb import TimescaleDBEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.postgres")
|
||||
|
||||
from testcontainers.community.postgres import PostgresContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with PostgresContainer("timescale/timescaledb:2.29.2-pg16") as container:
|
||||
yield create_engine(container.get_connection_url())
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
TimescaleDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector;
|
||||
this exercises that against actual server-reported column metadata
|
||||
rather than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = TimescaleDBEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = TimescaleDBEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,104 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.trino against a real Trino instance, spun up on
|
||||
demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
Only Presto is covered by existing docker-compose-based integration CI;
|
||||
Trino, despite sharing lineage with Presto, is not.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.trino import TrinoEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.community.trino")
|
||||
|
||||
from testcontainers.community.trino import TrinoContainer # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
with TrinoContainer() as container:
|
||||
# TrinoContainer.get_connection_url() (testcontainers 4.15.0) returns
|
||||
# the container-internal port (e.g. 8080) instead of the Docker-
|
||||
# mapped host port, so the URL it builds cannot actually connect.
|
||||
# Build it manually with get_exposed_port() instead. Filed upstream:
|
||||
# https://github.com/testcontainers/testcontainers-python/issues
|
||||
url = (
|
||||
f"trino://{container.user}@{container.get_container_host_ip()}"
|
||||
f":{container.get_exposed_port(container.port)}/memory/default"
|
||||
)
|
||||
yield create_engine(url)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. This is the exact bug class in apache/superset#42899,
|
||||
where Trino emitted OFFSET before LIMIT for paginated queries -- a
|
||||
dialect-compiler bug invisible to mocked tests, only catchable by
|
||||
actually executing the compiled SQL.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
TrinoEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = TrinoEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = TrinoEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,148 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.ydb against a real YDB instance, spun up on demand
|
||||
via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
YDB has no dedicated testcontainers module, so this uses a generic
|
||||
DockerContainer against the official `ydbplatform/local-ydb` image, which
|
||||
needs no auth for local/anonymous access -- YDBEngineSpec's own
|
||||
`sqlalchemy_uri_placeholder` ("ydb://{host}:{port}/{database_name}") has
|
||||
no username/password at all.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.ydb import YDBEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
require_driver("ydb_sqlalchemy")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import PortWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
GRPC_PORT = 2136
|
||||
DATABASE = "/local"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("ydbplatform/local-ydb")
|
||||
container.with_exposed_ports(GRPC_PORT)
|
||||
# YDB's gRPC client does endpoint discovery: it asks the server for its
|
||||
# "real" endpoints and reconnects to whatever comes back, rather than
|
||||
# just using the address it was originally given. By default that's
|
||||
# the container's own internal Docker hostname (e.g. "6abbb4bb0ab7"),
|
||||
# which isn't reachable from the host. Binding the same port number on
|
||||
# the host as inside the container, plus advertising "localhost" as
|
||||
# the container's own hostname, makes the discovered endpoint
|
||||
# ("localhost:2136") actually resolve to something reachable.
|
||||
container.with_bind_ports(GRPC_PORT, GRPC_PORT)
|
||||
container.with_kwargs(hostname="localhost")
|
||||
container.with_env("YDB_USE_IN_MEMORY_PDISKS", "true")
|
||||
container.waiting_for(PortWaitStrategy(GRPC_PORT))
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(GRPC_PORT)
|
||||
eng = create_engine(f"yql://{host}:{port}{DATABASE}")
|
||||
|
||||
# The gRPC port opens, and even a bare SELECT succeeds, before YDB's
|
||||
# storage pools are fully initialized -- an actual CREATE TABLE can
|
||||
# still fail with "database doesn't have storage pools at all to
|
||||
# create tablet channels" (confirmed on a real instance). Probe with
|
||||
# metadata.create_all()/drop_all() specifically, the same call the
|
||||
# real tests below make: a raw `text("CREATE TABLE ...")` hits a
|
||||
# separate, unrelated error ("Scheme operations cannot be executed
|
||||
# inside transaction") that create_all()'s own DDL execution path
|
||||
# doesn't, even with AUTOCOMMIT set on a manually-opened connection.
|
||||
probe_metadata = MetaData()
|
||||
SATable("pilot_ready", probe_metadata, Column("id", Integer, primary_key=True))
|
||||
last_error: Exception | None = None
|
||||
for _ in range(30):
|
||||
try:
|
||||
probe_metadata.create_all(eng)
|
||||
probe_metadata.drop_all(eng)
|
||||
break
|
||||
except Exception as ex: # noqa: BLE001 -- retry on any not-ready-yet error
|
||||
last_error = ex
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"YDB never became ready to create and use a table"
|
||||
) from last_error
|
||||
|
||||
yield eng
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
YDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = YDBEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = YDBEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -0,0 +1,119 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests db_engine_specs.yugabytedb against a real YugabyteDB instance, spun
|
||||
up on demand via testcontainers. Run via .github/workflows/testcontainers.yml.
|
||||
|
||||
YugabyteDB's YSQL layer is fully Postgres-wire compatible (postgresql+
|
||||
psycopg2, port 5433), but the image doesn't ship a plain `psql` binary
|
||||
(only its own `ysqlsh`), so testcontainers' PostgresContainer can't be
|
||||
reused directly -- its built-in readiness check execs `psql`, which would
|
||||
fail here. This uses a generic DockerContainer, starting the node via
|
||||
`yugabyted start --background=false` and waiting for yugabyted's own final
|
||||
startup message instead.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
create_engine,
|
||||
inspect,
|
||||
Integer,
|
||||
MetaData,
|
||||
Table as SATable,
|
||||
)
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from superset.db_engine_specs.yugabytedb import YugabyteDBEngineSpec
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils.core import GenericDataType
|
||||
|
||||
pytestmark = pytest.mark.testcontainers
|
||||
|
||||
from ._driver import require_driver # noqa: E402
|
||||
|
||||
require_driver("testcontainers.core.container")
|
||||
|
||||
from testcontainers.core.container import DockerContainer # noqa: E402
|
||||
from testcontainers.core.wait_strategies import LogMessageWaitStrategy # noqa: E402
|
||||
|
||||
from ._pagination import ( # noqa: E402
|
||||
assert_paginated_query_returns_correct_rows_in_order,
|
||||
)
|
||||
|
||||
YSQL_PORT = 5433
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> Iterator[Engine]:
|
||||
container = DockerContainer("yugabytedb/yugabyte:latest")
|
||||
container.with_exposed_ports(YSQL_PORT)
|
||||
container.with_command("bin/yugabyted start --background=false")
|
||||
container.waiting_for(
|
||||
LogMessageWaitStrategy(
|
||||
re.compile("Data placement constraint successfully verified")
|
||||
)
|
||||
)
|
||||
|
||||
with container:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(YSQL_PORT)
|
||||
# Default single-node credentials/database, per yugabyted's own
|
||||
# documented quickstart defaults -- no env vars needed to set them.
|
||||
yield create_engine(
|
||||
f"postgresql+psycopg2://yugabyte:yugabyte@{host}:{port}/yugabyte"
|
||||
)
|
||||
|
||||
|
||||
def test_paginated_query_returns_correct_rows_in_order(engine: Engine) -> None:
|
||||
"""
|
||||
A plain SQLAlchemy Core LIMIT/OFFSET query, compiled and executed against
|
||||
a real instance. Mocked tests cannot catch a dialect compiling this
|
||||
incorrectly (see apache/superset#42899, where Trino emitted OFFSET
|
||||
before LIMIT) -- only real execution can.
|
||||
"""
|
||||
assert_paginated_query_returns_correct_rows_in_order(engine)
|
||||
|
||||
|
||||
def test_get_columns_maps_native_types(engine: Engine) -> None:
|
||||
"""
|
||||
YugabyteDBEngineSpec.get_columns wraps a real SQLAlchemy Inspector; this
|
||||
exercises that against actual server-reported column metadata rather
|
||||
than a mocked Inspector.
|
||||
"""
|
||||
metadata = MetaData()
|
||||
SATable(
|
||||
"pilot_types",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("amount", Integer),
|
||||
)
|
||||
metadata.create_all(engine)
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = YugabyteDBEngineSpec.get_columns(inspector, Table("pilot_types"))
|
||||
|
||||
by_name = {col["column_name"]: col for col in columns}
|
||||
assert set(by_name) == {"id", "amount"}
|
||||
for col in by_name.values():
|
||||
spec = YugabyteDBEngineSpec.get_column_spec(str(col["type"]))
|
||||
assert spec is not None
|
||||
assert spec.generic_type == GenericDataType.NUMERIC
|
||||
assert isinstance(spec.sqla_type, Integer)
|
||||
@@ -133,3 +133,55 @@ def test_import_database_with_password_in_config(
|
||||
uuid = configs["databases/examples.yaml"]["uuid"]
|
||||
database = db.session.query(Database).filter_by(uuid=uuid).one()
|
||||
assert database.password == "yaml_password" # noqa: S105
|
||||
|
||||
|
||||
def test_transient_schema_listing_failure_is_recovered_by_retry(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""
|
||||
A single transient failure while listing schemas for a connection must
|
||||
not permanently discard the schema_access grant for a schema that
|
||||
appeared on the live connection since the last successful sync. Before
|
||||
this fix, ``add_permissions()`` gave up on a catalog after one failed
|
||||
attempt at ``get_all_schema_names()`` -- even though the exception
|
||||
caught (``GenericDBException``, a bare alias for the built-in
|
||||
``Exception``) does not distinguish a genuinely unlistable catalog from
|
||||
a one-off connector hiccup -- so a schema that needed a first-time grant
|
||||
during the failing attempt was never granted, silently, while the
|
||||
import still reported success.
|
||||
"""
|
||||
from superset import db, security_manager
|
||||
from superset.commands.database.importers.v1 import ImportDatabasesCommand
|
||||
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
||||
from superset.models.core import Database
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=True)
|
||||
add_permission_view_menu = mocker.spy(security_manager, "add_permission_view_menu")
|
||||
|
||||
engine = db.session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
configs = copy.deepcopy(databases_config)
|
||||
|
||||
# First import (existing=None): only "old_schema" exists on the live
|
||||
# target connection.
|
||||
mocker.patch.object(
|
||||
SqliteEngineSpec, "get_schema_names", return_value={"old_schema"}
|
||||
)
|
||||
ImportDatabasesCommand._import(copy.deepcopy(configs))
|
||||
add_permission_view_menu.reset_mock()
|
||||
|
||||
# A new schema now exists, but the first attempt to list schemas for
|
||||
# this connection hits a transient failure (e.g. a driver hiccup right
|
||||
# after the schema was created). Re-importing with overwrite=True --
|
||||
# the only way to legitimately re-import an existing connection -- must
|
||||
# still end up granting the new schema once the retry succeeds.
|
||||
mocker.patch.object(
|
||||
SqliteEngineSpec,
|
||||
"get_schema_names",
|
||||
side_effect=[Exception("transient driver error"), {"old_schema", "new_schema"}],
|
||||
)
|
||||
ImportDatabasesCommand._import(copy.deepcopy(configs), overwrite=True)
|
||||
|
||||
add_permission_view_menu.assert_any_call("schema_access", "[examples].[new_schema]")
|
||||
|
||||
@@ -296,7 +296,14 @@ def _stub_run_environment(mocker: MockerFixture) -> MagicMock:
|
||||
)
|
||||
db_mock = mocker.patch("superset.commands.database.uploaders.base.db")
|
||||
# No visible dataset over the target table.
|
||||
db_mock.session.query.return_value.filter.return_value.one_or_none.return_value = (
|
||||
None
|
||||
)
|
||||
db_mock.session.query.return_value.filter_by.return_value.one_or_none.return_value = None # noqa: E501
|
||||
mocker.patch(
|
||||
"superset.commands.database.uploaders.base.or_",
|
||||
side_effect=lambda *args: mocker.MagicMock(),
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
@@ -368,3 +375,57 @@ def test_run_proceeds_when_no_soft_deleted_twin(
|
||||
)
|
||||
command.run()
|
||||
reader.read.assert_called_once()
|
||||
|
||||
|
||||
def test_run_sets_default_catalog_on_dataset_creation(
|
||||
app_context: None, mocker: MockerFixture
|
||||
) -> None:
|
||||
"""UploadCommand sets default catalog on newly created dataset."""
|
||||
model = _stub_run_environment(mocker)
|
||||
model.get_default_catalog.return_value = "default_catalog"
|
||||
mocker.patch(
|
||||
"superset.daos.dataset.DatasetDAO.find_soft_deleted_logical_duplicate",
|
||||
return_value=None,
|
||||
)
|
||||
sqla_table_mock = mocker.patch(
|
||||
"superset.commands.database.uploaders.base.SqlaTable",
|
||||
return_value=MagicMock(),
|
||||
)
|
||||
mocker.patch(
|
||||
"superset.commands.database.uploaders.base.get_user",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
reader = MagicMock()
|
||||
command = UploadCommand(
|
||||
model_id=1, table_name="t", file=_file(b"x"), schema="public", reader=reader
|
||||
)
|
||||
command.run()
|
||||
|
||||
sqla_table_mock.assert_called_once()
|
||||
assert sqla_table_mock.call_args.kwargs.get("catalog") == "default_catalog"
|
||||
|
||||
|
||||
def test_run_updates_catalog_on_existing_dataset_with_none_catalog(
|
||||
app_context: None, mocker: MockerFixture
|
||||
) -> None:
|
||||
"""UploadCommand updates catalog on an existing dataset if catalog was None."""
|
||||
model = _stub_run_environment(mocker)
|
||||
model.get_default_catalog.return_value = "default_catalog"
|
||||
|
||||
existing_table = MagicMock()
|
||||
existing_table.catalog = None
|
||||
|
||||
db_mock = mocker.patch("superset.commands.database.uploaders.base.db")
|
||||
db_mock.session.query.return_value.filter.return_value.one_or_none.return_value = (
|
||||
existing_table
|
||||
)
|
||||
|
||||
reader = MagicMock()
|
||||
command = UploadCommand(
|
||||
model_id=1, table_name="t", file=_file(b"x"), schema="public", reader=reader
|
||||
)
|
||||
command.run()
|
||||
|
||||
assert existing_table.catalog == "default_catalog"
|
||||
existing_table.fetch_metadata.assert_called_once()
|
||||
|
||||
@@ -348,6 +348,71 @@ def test_import_passes_ignore_permissions_to_all_importers(
|
||||
assert mock_import_dashboard.call_args[1].get("ignore_permissions") is True
|
||||
|
||||
|
||||
@patch(
|
||||
"superset.commands.importers.v1.examples.safe_insert_dashboard_chart_relationships"
|
||||
)
|
||||
@patch("superset.commands.importers.v1.examples.import_dataset")
|
||||
@patch("superset.commands.importers.v1.examples.import_database")
|
||||
def test_import_dedupes_datasets_with_same_uuid(
|
||||
mock_import_db,
|
||||
mock_import_dataset,
|
||||
mock_safe_insert,
|
||||
):
|
||||
"""_import() must import a given dataset uuid at most once per run.
|
||||
|
||||
Two example folders can ship a dataset config for the same
|
||||
underlying table with an identical uuid (e.g. "world_health" and
|
||||
"misc_charts" both shipping a config for "wb_health_population").
|
||||
Importing it twice repeats the same column/metric sync for no
|
||||
benefit.
|
||||
"""
|
||||
from superset.commands.importers.v1.examples import ImportExamplesCommand
|
||||
|
||||
db_uuid = "a2dc77af-e654-49bb-b321-40f6b559a1ee"
|
||||
dataset_uuid = "69e9de42-fe7f-4948-946a-f7913227aee8"
|
||||
|
||||
mock_db_obj = MagicMock()
|
||||
mock_db_obj.uuid = db_uuid
|
||||
mock_db_obj.id = 1
|
||||
mock_import_db.return_value = mock_db_obj
|
||||
|
||||
mock_dataset_obj = MagicMock()
|
||||
mock_dataset_obj.uuid = dataset_uuid
|
||||
mock_dataset_obj.id = 10
|
||||
mock_dataset_obj.table_name = "wb_health_population"
|
||||
mock_import_dataset.return_value = mock_dataset_obj
|
||||
|
||||
configs = {
|
||||
"databases/examples.yaml": {
|
||||
"uuid": db_uuid,
|
||||
"database_name": "examples",
|
||||
"sqlalchemy_uri": "sqlite:///test.db",
|
||||
},
|
||||
"datasets/examples/world_health.yaml": {
|
||||
"uuid": dataset_uuid,
|
||||
"table_name": "wb_health_population",
|
||||
"database_uuid": db_uuid,
|
||||
"schema": None,
|
||||
"sql": None,
|
||||
},
|
||||
"datasets/examples/wb_health_population.yaml": {
|
||||
"uuid": dataset_uuid,
|
||||
"table_name": "wb_health_population",
|
||||
"database_uuid": db_uuid,
|
||||
"schema": None,
|
||||
"sql": None,
|
||||
},
|
||||
}
|
||||
|
||||
with patch(
|
||||
"superset.commands.importers.v1.examples.get_example_default_schema",
|
||||
return_value=None,
|
||||
):
|
||||
ImportExamplesCommand._import(configs)
|
||||
|
||||
mock_import_dataset.assert_called_once()
|
||||
|
||||
|
||||
def test_normalize_dataset_schema_converts_main_to_null():
|
||||
"""SQLite 'main' schema must be normalized to null in YAML content.
|
||||
|
||||
|
||||
@@ -146,6 +146,11 @@ def test_query_dao_stop_query_not_found(
|
||||
|
||||
db.session.add(database)
|
||||
db.session.add(query_obj)
|
||||
# Committed (not just autoflushed) since QueryDAO.stop_query() is now
|
||||
# wrapped in @transaction, which rolls back the session on the
|
||||
# QueryNotFoundException raised below -- an uncommitted insert would be
|
||||
# discarded along with it.
|
||||
db.session.commit()
|
||||
|
||||
mocker.patch("superset.sql_lab.cancel_query", return_value=False)
|
||||
|
||||
@@ -228,6 +233,11 @@ def test_query_dao_stop_query_failed(
|
||||
|
||||
db.session.add(database)
|
||||
db.session.add(query_obj)
|
||||
# Committed (not just autoflushed) since QueryDAO.stop_query() is now
|
||||
# wrapped in @transaction, which rolls back the session on the
|
||||
# SupersetCancelQueryException raised below -- an uncommitted insert
|
||||
# would be discarded along with it.
|
||||
db.session.commit()
|
||||
|
||||
mocker.patch("superset.sql_lab.cancel_query", return_value=False)
|
||||
|
||||
@@ -314,6 +324,11 @@ def test_query_dao_stop_query_wrong_user(
|
||||
|
||||
db.session.add(database)
|
||||
db.session.add(query_obj)
|
||||
# Committed (not just autoflushed) since QueryDAO.stop_query() is now
|
||||
# wrapped in @transaction, which rolls back the session on the
|
||||
# QueryNotFoundException raised below -- an uncommitted insert would be
|
||||
# discarded along with it.
|
||||
db.session.commit()
|
||||
|
||||
# Simulate a different user (user 2) attempting to stop user 1's query
|
||||
mocker.patch("superset.daos.query.get_user_id", return_value=2)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -344,6 +344,129 @@ def test_import_database_with_masked_encrypted_extra_existing_db(
|
||||
assert encrypted["credentials_info"]["private_key"] != PASSWORD_MASK
|
||||
|
||||
|
||||
def test_import_database_existing_no_overwrite_backfills_permissions(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""
|
||||
``import_database(config, overwrite=False)`` is called unconditionally by
|
||||
the chart, dataset, and saved-query bundle importers
|
||||
(``superset/commands/chart/importers/v1/__init__.py``,
|
||||
``superset/commands/dataset/importers/v1/__init__.py``, and
|
||||
``superset/commands/query/importers/v1/__init__.py``), and by the
|
||||
dashboard importer (``superset/commands/dashboard/importers/v1/__init__.py``)
|
||||
whenever both ``overwrite``/``overwrite_all`` aren't simultaneously
|
||||
``True`` -- the common case -- whenever the bundle references a
|
||||
database, including one that already exists in the target, eg because a
|
||||
prior import already created it. Before this fix, the early return for
|
||||
an existing, non-overwritten database skipped ``add_permissions()``
|
||||
entirely, so a schema added to the live connection since that database
|
||||
was first imported would never get a first-time grant through this path
|
||||
either.
|
||||
"""
|
||||
from superset import security_manager
|
||||
from superset.commands.database.importers.v1.utils import import_database
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=True)
|
||||
mock_add_permissions = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.add_permissions"
|
||||
)
|
||||
|
||||
engine = db.session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
existing = import_database(config)
|
||||
mock_add_permissions.reset_mock()
|
||||
|
||||
again = copy.deepcopy(database_config)
|
||||
result = import_database(again, overwrite=False)
|
||||
|
||||
assert result.id == existing.id
|
||||
mock_add_permissions.assert_called_once_with(existing)
|
||||
|
||||
|
||||
def test_import_database_existing_no_overwrite_no_permission_skips_backfill(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""
|
||||
The permission backfill added for the existing/no-overwrite branch must
|
||||
stay behind the same ``can_write`` gate as every other permission-view
|
||||
creation in ``import_database()`` -- a principal without database-write
|
||||
access must not trigger it just by importing a
|
||||
chart/dataset/saved-query/dashboard bundle that happens to reference an
|
||||
existing database.
|
||||
"""
|
||||
from superset import security_manager
|
||||
from superset.commands.database.importers.v1.utils import import_database
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=True)
|
||||
mock_add_permissions = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.add_permissions"
|
||||
)
|
||||
|
||||
engine = db.session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
existing = import_database(config)
|
||||
mock_add_permissions.reset_mock()
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=False)
|
||||
|
||||
again = copy.deepcopy(database_config)
|
||||
result = import_database(again, overwrite=False)
|
||||
|
||||
assert result.id == existing.id
|
||||
mock_add_permissions.assert_not_called()
|
||||
|
||||
|
||||
def test_import_database_existing_no_overwrite_backfill_oauth2_redirect_is_nonfatal(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""
|
||||
The permission backfill on the existing/no-overwrite branch must tolerate
|
||||
an ``OAuth2RedirectError`` from ``add_permissions()`` the same way the
|
||||
fresh-import path already does (see
|
||||
``test_import_database_oauth2_redirect_is_nonfatal``) -- logged, not
|
||||
propagated, so the import still returns the existing database normally.
|
||||
"""
|
||||
from superset import security_manager
|
||||
from superset.commands.database.importers.v1.utils import import_database
|
||||
from superset.exceptions import OAuth2RedirectError
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=True)
|
||||
mock_add_permissions = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.add_permissions"
|
||||
)
|
||||
|
||||
engine = db.session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
existing = import_database(config)
|
||||
mock_add_permissions.reset_mock()
|
||||
mock_add_permissions.side_effect = OAuth2RedirectError(
|
||||
url="https://oauth.example.com/authorize",
|
||||
tab_id="abc-123",
|
||||
redirect_uri="https://superset.example.com/callback",
|
||||
)
|
||||
|
||||
again = copy.deepcopy(database_config)
|
||||
result = import_database(again, overwrite=False)
|
||||
|
||||
assert result.id == existing.id
|
||||
mock_add_permissions.assert_called_once_with(existing)
|
||||
|
||||
|
||||
def test_import_database_oauth2_redirect_is_nonfatal(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
@@ -378,6 +501,45 @@ def test_import_database_oauth2_redirect_is_nonfatal(
|
||||
mock_add_perms.assert_called_once_with(database)
|
||||
|
||||
|
||||
def test_import_database_dbapi_error_during_backfill_is_nonfatal(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""
|
||||
``add_permissions()`` calls ``get_all_catalog_names()`` for catalog
|
||||
discovery outside of its own per-catalog error handling, so a mapped
|
||||
DBAPI error from that call -- not just a connection failure -- must be
|
||||
tolerated the same way an ``OAuth2RedirectError`` already is, instead of
|
||||
failing the whole import.
|
||||
"""
|
||||
from superset import security_manager
|
||||
from superset.commands.database.importers.v1.utils import import_database
|
||||
from superset.db_engine_specs.exceptions import SupersetDBAPIProgrammingError
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
mocker.patch.object(security_manager, "can_access", return_value=True)
|
||||
mock_add_permissions = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.add_permissions"
|
||||
)
|
||||
|
||||
engine = db.session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
existing = import_database(config)
|
||||
mock_add_permissions.reset_mock()
|
||||
mock_add_permissions.side_effect = SupersetDBAPIProgrammingError(
|
||||
"permission denied for catalog discovery"
|
||||
)
|
||||
|
||||
again = copy.deepcopy(database_config)
|
||||
result = import_database(again, overwrite=False)
|
||||
|
||||
assert result.id == existing.id
|
||||
mock_add_permissions.assert_called_once_with(existing)
|
||||
|
||||
|
||||
def test_import_datasources_cli_encrypts_password(
|
||||
mocker: MockerFixture,
|
||||
session: Session,
|
||||
|
||||
@@ -78,14 +78,20 @@ def test_add_permissions_get_default_catalog(mocker: MockerFixture):
|
||||
|
||||
def test_add_permissions_handle_failures(mocker: MockerFixture) -> None:
|
||||
"""
|
||||
Test adding permissions to a database when it's created in case
|
||||
the request to get all schemas for one fo the catalogs fail.
|
||||
Test adding permissions to a database when it's created in case the
|
||||
request to get all schemas for one of the catalogs keeps failing --
|
||||
each failing catalog is retried once before being given up on.
|
||||
"""
|
||||
database = mocker.MagicMock()
|
||||
database.database_name = "my_db"
|
||||
database.db_engine_spec.supports_catalog = True
|
||||
database.get_all_catalog_names.return_value = ["catalog1", "catalog2", "catalog3"]
|
||||
database.get_all_schema_names.side_effect = [["schema1"], Exception, ["schema3"]]
|
||||
database.get_all_schema_names.side_effect = [
|
||||
["schema1"],
|
||||
Exception,
|
||||
Exception,
|
||||
["schema3"],
|
||||
]
|
||||
add_permission_view_menu = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.security_manager."
|
||||
"add_permission_view_menu"
|
||||
@@ -102,3 +108,119 @@ def test_add_permissions_handle_failures(mocker: MockerFixture) -> None:
|
||||
mocker.call("schema_access", "[my_db].[catalog3].[schema3]"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_add_permissions_retries_transient_failure(mocker: MockerFixture) -> None:
|
||||
"""
|
||||
A catalog whose schema listing fails only once (eg a transient driver
|
||||
hiccup) must still get its schema permissions granted via the retry,
|
||||
instead of being permanently skipped like a genuinely unlistable
|
||||
catalog would be.
|
||||
"""
|
||||
database = mocker.MagicMock()
|
||||
database.database_name = "my_db"
|
||||
database.db_engine_spec.supports_catalog = True
|
||||
database.get_all_catalog_names.return_value = ["catalog1", "catalog2"]
|
||||
database.get_all_schema_names.side_effect = [
|
||||
["schema1"],
|
||||
Exception,
|
||||
["schema2"],
|
||||
]
|
||||
add_permission_view_menu = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.security_manager."
|
||||
"add_permission_view_menu"
|
||||
)
|
||||
|
||||
add_permissions(database)
|
||||
|
||||
add_permission_view_menu.assert_has_calls(
|
||||
[
|
||||
mocker.call("catalog_access", "[my_db].[catalog1]"),
|
||||
mocker.call("catalog_access", "[my_db].[catalog2]"),
|
||||
mocker.call("schema_access", "[my_db].[catalog1].[schema1]"),
|
||||
mocker.call("schema_access", "[my_db].[catalog2].[schema2]"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_add_permissions_does_not_retry_oauth2_redirect(mocker: MockerFixture) -> None:
|
||||
"""
|
||||
An ``OAuth2RedirectError`` from the schema-listing call is a control-flow
|
||||
signal, not a transient failure: retrying it would just kick off a
|
||||
second, redundant OAuth2 authorization redirect for the same catalog.
|
||||
"""
|
||||
from superset.exceptions import OAuth2RedirectError
|
||||
|
||||
database = mocker.MagicMock()
|
||||
database.database_name = "my_db"
|
||||
database.db_engine_spec.supports_catalog = True
|
||||
database.get_all_catalog_names.return_value = ["catalog1", "catalog2"]
|
||||
oauth2_error = OAuth2RedirectError(
|
||||
url="https://oauth.example.com/authorize",
|
||||
tab_id="abc-123",
|
||||
redirect_uri="https://superset.example.com/callback",
|
||||
)
|
||||
database.get_all_schema_names.side_effect = [oauth2_error, ["schema2"]]
|
||||
add_permission_view_menu = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.security_manager."
|
||||
"add_permission_view_menu"
|
||||
)
|
||||
|
||||
add_permissions(database)
|
||||
|
||||
assert database.get_all_schema_names.call_count == 2
|
||||
add_permission_view_menu.assert_has_calls(
|
||||
[
|
||||
mocker.call("catalog_access", "[my_db].[catalog1]"),
|
||||
mocker.call("catalog_access", "[my_db].[catalog2]"),
|
||||
mocker.call("schema_access", "[my_db].[catalog2].[schema2]"),
|
||||
]
|
||||
)
|
||||
# [catalog1] must never be retried: the OAuth2 redirect aborts it after
|
||||
# a single attempt, same as a genuinely unlistable catalog would be.
|
||||
assert add_permission_view_menu.call_count == 3
|
||||
|
||||
|
||||
def test_add_permissions_tolerates_failure_creating_permission_view(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
A failure while granting a schema's permission (as opposed to while
|
||||
listing the catalog's schemas) must be tolerated the same way it was
|
||||
before the schema-listing retry was introduced: the rest of that
|
||||
catalog is abandoned, but the next catalog is still processed. The
|
||||
retry is scoped to the schema-listing call only, so it must not narrow
|
||||
this pre-existing tolerance.
|
||||
"""
|
||||
database = mocker.MagicMock()
|
||||
database.database_name = "my_db"
|
||||
database.db_engine_spec.supports_catalog = True
|
||||
database.get_all_catalog_names.return_value = ["catalog1", "catalog2"]
|
||||
database.get_all_schema_names.side_effect = [
|
||||
["schema1a", "schema1b"],
|
||||
["schema2"],
|
||||
]
|
||||
add_permission_view_menu = mocker.patch(
|
||||
"superset.commands.database.importers.v1.utils.security_manager."
|
||||
"add_permission_view_menu"
|
||||
)
|
||||
add_permission_view_menu.side_effect = [
|
||||
None, # catalog_access [catalog1]
|
||||
None, # catalog_access [catalog2]
|
||||
Exception, # schema_access [catalog1].[schema1a] -- fails
|
||||
None, # schema_access [catalog2].[schema2]
|
||||
]
|
||||
|
||||
add_permissions(database)
|
||||
|
||||
add_permission_view_menu.assert_has_calls(
|
||||
[
|
||||
mocker.call("catalog_access", "[my_db].[catalog1]"),
|
||||
mocker.call("catalog_access", "[my_db].[catalog2]"),
|
||||
mocker.call("schema_access", "[my_db].[catalog1].[schema1a]"),
|
||||
mocker.call("schema_access", "[my_db].[catalog2].[schema2]"),
|
||||
]
|
||||
)
|
||||
# [catalog1].[schema1b] must never be attempted: the exception raised
|
||||
# while granting [schema1a] aborts the rest of catalog1, same as before.
|
||||
assert add_permission_view_menu.call_count == 4
|
||||
|
||||
@@ -172,6 +172,56 @@ def test_get_available_engine_specs_supports_sqlalchemy_2_native_dialect(
|
||||
assert available[MySQLEngineSpec] == {"mysqldb"}
|
||||
|
||||
|
||||
def test_get_available_engine_specs_restores_compiler_operators(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
A third-party ``sqlalchemy.dialects`` entry point that mutates SQLAlchemy's
|
||||
shared, process-global ``compiler.OPERATORS`` mapping on import (as
|
||||
``sqlalchemy-monetdb`` does, in place, rather than subclassing) must not be
|
||||
allowed to leak that change into every other dialect for the rest of the
|
||||
process.
|
||||
|
||||
Regression test: enumerating a real "monetdb" entry point here (to build the
|
||||
"available databases" list) silently changed ``!=`` rendering to ``<>`` for
|
||||
postgres/mysql/sqlite/etc. too, for the remainder of the process.
|
||||
"""
|
||||
from sqlalchemy.sql import compiler as sqla_compiler, operators
|
||||
|
||||
mocker.patch(
|
||||
"superset.db_engine_specs.load_engine_specs",
|
||||
return_value=iter([]),
|
||||
)
|
||||
|
||||
pristine = dict(sqla_compiler.OPERATORS)
|
||||
assert pristine[operators.ne] != " <> "
|
||||
|
||||
class MisbehavingDialect(DefaultDialect):
|
||||
name = "misbehaving"
|
||||
driver = "misbehaving_driver"
|
||||
|
||||
def load_and_mutate_globally() -> type[MisbehavingDialect]:
|
||||
# Mirrors sqlalchemy-monetdb's `base.py`: grabs a reference to the
|
||||
# shared dict (not a copy) and mutates it in place.
|
||||
sqla_compiler.OPERATORS[operators.ne] = " <> "
|
||||
return MisbehavingDialect
|
||||
|
||||
entry_point = mocker.MagicMock()
|
||||
entry_point.name = "misbehaving"
|
||||
entry_point.load.side_effect = load_and_mutate_globally
|
||||
mocker.patch(
|
||||
"superset.db_engine_specs.entry_points",
|
||||
return_value=[entry_point],
|
||||
)
|
||||
|
||||
try:
|
||||
get_available_engine_specs()
|
||||
assert sqla_compiler.OPERATORS[operators.ne] == pristine[operators.ne]
|
||||
finally:
|
||||
sqla_compiler.OPERATORS.clear()
|
||||
sqla_compiler.OPERATORS.update(pristine)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"app",
|
||||
[{"DBS_AVAILABLE_DENYLIST": {"databricks": {"pyhive", "pyodbc"}}}],
|
||||
|
||||
@@ -37,6 +37,7 @@ from superset.mcp_service.chart.schemas import (
|
||||
GaugeChartConfig,
|
||||
GenerateChartResponse,
|
||||
LegendConfig,
|
||||
MixedTimeseriesChartConfig,
|
||||
TableChartConfig,
|
||||
UpdateChartRequest,
|
||||
XYChartConfig,
|
||||
@@ -44,6 +45,7 @@ from superset.mcp_service.chart.schemas import (
|
||||
from superset.mcp_service.chart.tool.update_chart import (
|
||||
_build_preview_form_data,
|
||||
_build_update_payload,
|
||||
_inherited_state_invalid_keys,
|
||||
)
|
||||
from superset.utils import json
|
||||
|
||||
@@ -739,6 +741,288 @@ class TestBuildUpdatePayload:
|
||||
# query_context must be cleared so get_chart_data uses updated params
|
||||
assert result["query_context"] is None
|
||||
|
||||
def test_config_update_preserves_unrelated_mixed_timeseries_settings(self):
|
||||
"""Save payload retains settings outside the simplified config schema."""
|
||||
config = MixedTimeseriesChartConfig(
|
||||
x=ColumnRef(name="ds"),
|
||||
y=[ColumnRef(name="new_primary", aggregate="SUM")],
|
||||
y_secondary=[ColumnRef(name="new_secondary", aggregate="SUM")],
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Year-over-year metrics",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "mixed_timeseries",
|
||||
"time_compare": ["1 year ago"],
|
||||
"comparison_type_b": "percentage",
|
||||
"y_axis_format": ",.2f",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
result = _build_update_payload(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
saved_form_data = json.loads(result["params"])
|
||||
assert saved_form_data["time_compare"] == ["1 year ago"]
|
||||
assert saved_form_data["comparison_type_b"] == "percentage"
|
||||
assert saved_form_data["y_axis_format"] == ",.2f"
|
||||
|
||||
@patch(
|
||||
"superset.mcp_service.chart.chart_utils.is_column_truly_temporal",
|
||||
return_value=True,
|
||||
)
|
||||
def test_temporal_update_preserves_omitted_non_temporal_filters(
|
||||
self, unused_temporal_mock
|
||||
) -> None:
|
||||
"""Generated time bindings do not replace omitted saved predicates."""
|
||||
country_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "US",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "country",
|
||||
}
|
||||
temporal_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
config = XYChartConfig(
|
||||
x=ColumnRef(name="ds"),
|
||||
y=[ColumnRef(name="revenue", aggregate="SUM")],
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Revenue",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "echarts_timeseries_line",
|
||||
"adhoc_filters": [country_filter, temporal_filter],
|
||||
"_mcp_dashboard_time_filter_subject": "ds",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
saved = _build_update_payload(request, chart, parsed_config=config)
|
||||
preview = _build_preview_form_data(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(saved, dict)
|
||||
assert isinstance(preview, dict)
|
||||
saved_filters = json.loads(saved["params"])["adhoc_filters"]
|
||||
assert saved_filters == [country_filter, temporal_filter]
|
||||
assert preview["adhoc_filters"] == saved_filters
|
||||
|
||||
def test_explicit_empty_group_by_clears_save_and_preview(self) -> None:
|
||||
config = XYChartConfig.model_validate(
|
||||
{
|
||||
"x": {"name": "ds"},
|
||||
"y": [{"name": "revenue", "aggregate": "SUM"}],
|
||||
"groupby": [],
|
||||
}
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Revenue by region",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "echarts_timeseries_line",
|
||||
"groupby": ["region"],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
saved = _build_update_payload(request, chart, parsed_config=config)
|
||||
preview = _build_preview_form_data(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(saved, dict)
|
||||
assert isinstance(preview, dict)
|
||||
assert "groupby" not in json.loads(saved["params"])
|
||||
assert "groupby" not in preview
|
||||
|
||||
def test_explicit_empty_sort_by_clears_save_and_preview(self) -> None:
|
||||
config = TableChartConfig.model_validate(
|
||||
{
|
||||
"columns": [{"name": "region"}],
|
||||
"order_by_cols": [],
|
||||
}
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Regions",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "table",
|
||||
"order_by_cols": ['["region", false]'],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
saved = _build_update_payload(request, chart, parsed_config=config)
|
||||
preview = _build_preview_form_data(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(saved, dict)
|
||||
assert isinstance(preview, dict)
|
||||
assert "order_by_cols" not in json.loads(saved["params"])
|
||||
assert "order_by_cols" not in preview
|
||||
|
||||
@patch.object(update_chart_module, "_inherited_state_invalid_keys")
|
||||
def test_dataset_rebind_only_preserves_compatible_state(
|
||||
self, mock_invalid_keys
|
||||
) -> None:
|
||||
config = TableChartConfig(columns=[ColumnRef(name="revenue")])
|
||||
request = UpdateChartRequest(identifier=1, config=config, dataset_id=9)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Revenue",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "table",
|
||||
"groupby": ["removed_column"],
|
||||
"custom_flag": True,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
mock_invalid_keys.return_value = {"groupby"}
|
||||
incompatible = _build_update_payload(request, chart, parsed_config=config)
|
||||
mock_invalid_keys.return_value = set()
|
||||
compatible = _build_update_payload(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(incompatible, dict)
|
||||
assert isinstance(compatible, dict)
|
||||
incompatible_params = json.loads(incompatible["params"])
|
||||
compatible_params = json.loads(compatible["params"])
|
||||
assert "groupby" not in incompatible_params
|
||||
assert incompatible_params["custom_flag"] is True
|
||||
assert compatible_params["groupby"] == ["removed_column"]
|
||||
assert compatible_params["custom_flag"] is True
|
||||
assert incompatible_params["datasource"] == "9__table"
|
||||
assert compatible_params["datasource"] == "9__table"
|
||||
|
||||
@patch(
|
||||
"superset.mcp_service.chart.validation.dataset_validator."
|
||||
"build_dataset_context_from_orm"
|
||||
)
|
||||
@patch("superset.daos.dataset.DatasetDAO.find_by_id")
|
||||
def test_dataset_rebind_explicit_clear_preserves_other_state(
|
||||
self, mock_find_dataset, mock_build_context
|
||||
) -> None:
|
||||
mock_find_dataset.return_value = Mock()
|
||||
mock_build_context.return_value = Mock(
|
||||
available_columns=[{"name": "revenue"}],
|
||||
available_metrics=[],
|
||||
)
|
||||
config = XYChartConfig.model_validate(
|
||||
{
|
||||
"x": {"name": "revenue"},
|
||||
"y": [{"name": "revenue", "aggregate": "SUM"}],
|
||||
"groupby": [],
|
||||
}
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config, dataset_id=9)
|
||||
chart = Mock(
|
||||
datasource_id=7,
|
||||
slice_name="Revenue",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "echarts_timeseries_line",
|
||||
"groupby": ["removed_column"],
|
||||
"custom_flag": True,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
result = _build_update_payload(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
params = json.loads(result["params"])
|
||||
assert "groupby" not in params
|
||||
assert params["custom_flag"] is True
|
||||
|
||||
@patch(
|
||||
"superset.mcp_service.chart.validation.dataset_validator."
|
||||
"build_dataset_context_from_orm"
|
||||
)
|
||||
@patch("superset.daos.dataset.DatasetDAO.find_by_id")
|
||||
def test_dataset_rebind_compatibility_uses_inherited_references(
|
||||
self, mock_find_dataset, mock_build_context
|
||||
) -> None:
|
||||
mock_find_dataset.return_value = Mock()
|
||||
mock_build_context.return_value = Mock(
|
||||
available_columns=[{"name": "revenue"}, {"name": "region"}],
|
||||
available_metrics=[],
|
||||
)
|
||||
config = TableChartConfig(columns=[ColumnRef(name="revenue")])
|
||||
new_form_data = {"viz_type": "table", "all_columns": ["revenue"]}
|
||||
|
||||
assert not _inherited_state_invalid_keys(
|
||||
{
|
||||
"viz_type": "table",
|
||||
"groupby": ["region"],
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"expressionType": "SIMPLE",
|
||||
"subject": "region",
|
||||
"operator": "==",
|
||||
"comparator": "US",
|
||||
}
|
||||
],
|
||||
},
|
||||
new_form_data,
|
||||
config,
|
||||
9,
|
||||
)
|
||||
assert _inherited_state_invalid_keys(
|
||||
{
|
||||
"viz_type": "table",
|
||||
"groupby": ["removed_column"],
|
||||
},
|
||||
new_form_data,
|
||||
config,
|
||||
9,
|
||||
) == {"groupby"}
|
||||
|
||||
def test_config_update_does_not_merge_settings_from_another_viz_type(self):
|
||||
"""Changing visualization types drops stale query-defining settings."""
|
||||
config = XYChartConfig(
|
||||
x=ColumnRef(name="ds"),
|
||||
y=[ColumnRef(name="revenue", aggregate="SUM")],
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config)
|
||||
chart = Mock(
|
||||
id=1,
|
||||
datasource_id=7,
|
||||
slice_name="Raw records",
|
||||
params=json.dumps(
|
||||
{
|
||||
"viz_type": "table",
|
||||
"query_mode": "raw",
|
||||
"all_columns": ["ds", "revenue"],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
result = _build_update_payload(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
saved_form_data = json.loads(result["params"])
|
||||
assert saved_form_data["viz_type"] == "echarts_timeseries_line"
|
||||
assert "query_mode" not in saved_form_data
|
||||
assert "all_columns" not in saved_form_data
|
||||
|
||||
def test_add_columns_preserves_existing_columns_and_metrics(self):
|
||||
"""An additive update does not require reconstructing the table."""
|
||||
request = UpdateChartRequest(
|
||||
@@ -1160,7 +1444,9 @@ class TestUpdateChartPreviewFirst:
|
||||
mock_chart.slice_name = "Existing Chart"
|
||||
mock_chart.viz_type = "table"
|
||||
mock_chart.uuid = "abc-123"
|
||||
mock_chart.params = '{"viz_type": "table", "datasource": "10__table"}'
|
||||
mock_chart.params = (
|
||||
'{"viz_type": "table", "datasource": "10__table", "custom_flag": true}'
|
||||
)
|
||||
mock_find_by_id.return_value = mock_chart
|
||||
|
||||
mock_check_access.return_value = DatasetValidationResult(
|
||||
@@ -1196,6 +1482,7 @@ class TestUpdateChartPreviewFirst:
|
||||
# Ensure the chart was NOT persisted
|
||||
mock_update_cmd_cls.assert_not_called()
|
||||
mock_create_preview.assert_called_once()
|
||||
assert mock_create_preview.call_args.args[1]["custom_flag"] is True
|
||||
|
||||
@patch.object(update_chart_module, "_create_preview_url", new_callable=Mock)
|
||||
@patch(
|
||||
@@ -1252,13 +1539,15 @@ class TestBuildPreviewFormData:
|
||||
chart.id = 42
|
||||
chart.datasource_id = 7
|
||||
chart.slice_name = "Existing"
|
||||
chart.params = '{"viz_type": "line", "custom_flag": true}'
|
||||
chart.params = '{"viz_type": "table", "custom_flag": true}'
|
||||
|
||||
result = _build_preview_form_data(request, chart, parsed_config=config)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
# Cross-viz updates do not inherit stale controls from the prior plugin.
|
||||
assert "custom_flag" not in result
|
||||
# A same-viz update keeps controls the config does not mention.
|
||||
# Cross-viz drops are covered by
|
||||
# test_config_update_does_not_merge_settings_from_another_viz_type.
|
||||
assert result["custom_flag"] is True
|
||||
# New config overrides existing keys
|
||||
assert result["viz_type"] == "table"
|
||||
# slice_id and datasource are always stamped onto the preview
|
||||
@@ -1560,7 +1849,7 @@ class TestUpdateChartSaveWithConfig:
|
||||
mock_chart.slice_name = "Pre-save"
|
||||
mock_chart.viz_type = "table"
|
||||
mock_chart.uuid = "uuid-77"
|
||||
mock_chart.params = '{"viz_type": "table"}'
|
||||
mock_chart.params = '{"viz_type": "table", "custom_flag": true}'
|
||||
mock_find_by_id.return_value = mock_chart
|
||||
|
||||
mock_check_access.return_value = DatasetValidationResult(
|
||||
@@ -1600,6 +1889,7 @@ class TestUpdateChartSaveWithConfig:
|
||||
# Verify query_context is cleared so get_chart_data uses updated params
|
||||
payload = mock_update_cmd_cls.call_args[0][1]
|
||||
assert payload["query_context"] is None
|
||||
assert json.loads(payload["params"])["custom_flag"] is True
|
||||
|
||||
# Verify form_data is returned in the response
|
||||
form_data = result.structured_content["form_data"]
|
||||
@@ -2214,7 +2504,7 @@ class TestBuildUpdatePayloadDatasetId:
|
||||
columns=[ColumnRef(name="col1")],
|
||||
)
|
||||
request = UpdateChartRequest(identifier=1, config=config, dataset_id=99)
|
||||
chart = Mock()
|
||||
chart = Mock(params='{"viz_type":"table","datasource":"10__table"}')
|
||||
chart.datasource_id = 10
|
||||
chart.slice_name = "Old Name"
|
||||
|
||||
@@ -2225,6 +2515,7 @@ class TestBuildUpdatePayloadDatasetId:
|
||||
assert result["datasource_type"] == "table"
|
||||
assert "params" in result
|
||||
assert "viz_type" in result
|
||||
assert json.loads(result["params"])["datasource"] == "99__table"
|
||||
|
||||
def test_config_without_dataset_does_not_include_datasource(self):
|
||||
"""When dataset_id is None, payload must NOT include datasource_id."""
|
||||
|
||||
@@ -21,6 +21,7 @@ Unit tests for update_chart_preview MCP tool
|
||||
|
||||
import importlib
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
@@ -28,8 +29,13 @@ from fastmcp import Client
|
||||
|
||||
from superset.extensions import feature_flag_manager
|
||||
from superset.mcp_service.app import mcp
|
||||
from superset.mcp_service.chart.chart_utils import (
|
||||
map_big_number_config,
|
||||
preserve_previous_adhoc_filters,
|
||||
)
|
||||
from superset.mcp_service.chart.schemas import (
|
||||
AxisConfig,
|
||||
BigNumberChartConfig,
|
||||
ColumnRef,
|
||||
FilterConfig,
|
||||
GaugeChartConfig,
|
||||
@@ -683,6 +689,287 @@ class TestUpdateChartPreview:
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_preserves_generated_temporal_filter_with_cached_filters(self) -> None:
|
||||
"""Cached filters are merged without replacing the temporal binding."""
|
||||
new_form_data = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
]
|
||||
}
|
||||
previous_form_data = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "North",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "region",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
previous_form_data,
|
||||
)
|
||||
|
||||
assert [filter_["subject"] for filter_ in new_form_data["adhoc_filters"]] == [
|
||||
"region",
|
||||
"ds",
|
||||
]
|
||||
|
||||
def test_cached_temporal_filter_takes_precedence_over_generated_default(
|
||||
self,
|
||||
) -> None:
|
||||
"""A cached chart-specific time range is not duplicated or reset."""
|
||||
new_form_data = {
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
]
|
||||
}
|
||||
cached_temporal_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "Last month",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{"adhoc_filters": [cached_temporal_filter]},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [cached_temporal_filter]
|
||||
|
||||
def test_preserves_filters_with_distinct_comparators(self) -> None:
|
||||
"""Filters that differ only by value are not deduplicated."""
|
||||
previous_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": 2020,
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": ">",
|
||||
"subject": "year",
|
||||
}
|
||||
generated_filter = {**previous_filter, "comparator": 2021}
|
||||
new_form_data = {"adhoc_filters": [generated_filter]}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{"adhoc_filters": [previous_filter]},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [previous_filter, generated_filter]
|
||||
|
||||
def test_replaces_cached_temporal_filter_when_column_changes(self) -> None:
|
||||
"""A newly selected temporal column replaces the cached binding."""
|
||||
new_temporal_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "created_at",
|
||||
}
|
||||
region_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "North",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "region",
|
||||
}
|
||||
previous_temporal_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
new_form_data: dict[str, Any] = {"adhoc_filters": [new_temporal_filter]}
|
||||
new_form_data["_mcp_dashboard_time_filter_subject"] = "created_at"
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{
|
||||
"adhoc_filters": [region_filter, previous_temporal_filter],
|
||||
"_mcp_dashboard_time_filter_subject": "ds",
|
||||
},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [
|
||||
region_filter,
|
||||
new_temporal_filter,
|
||||
]
|
||||
|
||||
def test_replaces_temporal_xy_binding_when_subject_changes(self) -> None:
|
||||
"""A temporal XY binding does not survive rebinding to a new subject."""
|
||||
previous_binding = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "event_time",
|
||||
}
|
||||
new_binding = {
|
||||
**previous_binding,
|
||||
"subject": "created_at",
|
||||
}
|
||||
new_form_data = {
|
||||
"adhoc_filters": [new_binding],
|
||||
"_mcp_dashboard_time_filter_subject": "created_at",
|
||||
}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{
|
||||
"adhoc_filters": [previous_binding],
|
||||
"_mcp_dashboard_time_filter_subject": "event_time",
|
||||
},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [new_binding]
|
||||
|
||||
def test_replaces_big_number_fallback_binding_when_subject_changes(self) -> None:
|
||||
"""A Big Number fallback binding is replaced by a selected subject."""
|
||||
dataset = Mock(
|
||||
main_dttm_col=None,
|
||||
columns=[Mock(column_name="order_date")],
|
||||
)
|
||||
config = BigNumberChartConfig(
|
||||
chart_type="big_number",
|
||||
metric=ColumnRef(name="revenue", aggregate="SUM"),
|
||||
)
|
||||
rebound_config = config.model_copy(update={"temporal_column": "created_at"})
|
||||
|
||||
with (
|
||||
patch(
|
||||
"superset.daos.dataset.DatasetDAO.find_by_id_or_uuid",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch(
|
||||
"superset.mcp_service.chart.chart_utils.is_column_truly_temporal",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
previous_form_data = map_big_number_config(config, dataset_id=42)
|
||||
new_form_data = map_big_number_config(rebound_config, dataset_id=42)
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
previous_form_data,
|
||||
)
|
||||
|
||||
assert previous_form_data["_mcp_dashboard_time_filter_subject"] == "order_date"
|
||||
assert new_form_data["_mcp_dashboard_time_filter_subject"] == "created_at"
|
||||
assert [filter_["subject"] for filter_ in new_form_data["adhoc_filters"]] == [
|
||||
"created_at"
|
||||
]
|
||||
|
||||
def test_removes_cached_temporal_filter_without_new_binding(self) -> None:
|
||||
"""A mapping without a temporal subject drops the cached binding."""
|
||||
region_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "North",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "==",
|
||||
"subject": "region",
|
||||
}
|
||||
previous_temporal_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
new_form_data: dict[str, Any] = {}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{
|
||||
"adhoc_filters": [region_filter, previous_temporal_filter],
|
||||
"_mcp_dashboard_time_filter_subject": "ds",
|
||||
},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [region_filter]
|
||||
|
||||
def test_preserves_user_temporal_filter_on_generated_subject(self) -> None:
|
||||
"""A user-authored range on the binding subject is not generated state."""
|
||||
generated_binding = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "No filter",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
user_filter = {
|
||||
"clause": "WHERE",
|
||||
"comparator": "Last month",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
}
|
||||
|
||||
new_form_data: dict[str, Any] = {}
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{
|
||||
"adhoc_filters": [generated_binding, user_filter],
|
||||
"_mcp_dashboard_time_filter_subject": "ds",
|
||||
},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [user_filter]
|
||||
|
||||
def test_rebinding_preserves_unrelated_cached_temporal_filter(self) -> None:
|
||||
"""Only the generated binding is replaced; user filters retain provenance."""
|
||||
previous_binding = {
|
||||
"expressionType": "SIMPLE",
|
||||
"clause": "WHERE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "ds",
|
||||
"comparator": "No filter",
|
||||
}
|
||||
unrelated_filter = {
|
||||
"expressionType": "SIMPLE",
|
||||
"clause": "WHERE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "processed_at",
|
||||
"comparator": "Last year",
|
||||
}
|
||||
new_binding = {
|
||||
"expressionType": "SIMPLE",
|
||||
"clause": "WHERE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": "created_at",
|
||||
"comparator": "No filter",
|
||||
}
|
||||
new_form_data = {
|
||||
"adhoc_filters": [new_binding],
|
||||
"_mcp_dashboard_time_filter_subject": "created_at",
|
||||
}
|
||||
|
||||
preserve_previous_adhoc_filters(
|
||||
new_form_data,
|
||||
{
|
||||
"adhoc_filters": [previous_binding, unrelated_filter],
|
||||
"_mcp_dashboard_time_filter_subject": "ds",
|
||||
},
|
||||
)
|
||||
|
||||
assert new_form_data["adhoc_filters"] == [unrelated_filter, new_binding]
|
||||
|
||||
@patch.object(update_chart_preview_module, "validate_and_compile")
|
||||
@patch.object(update_chart_preview_module, "has_dataset_access", return_value=True)
|
||||
@patch("superset.daos.dataset.DatasetDAO.find_by_id")
|
||||
|
||||
@@ -29,6 +29,8 @@ from superset.jinja_context import JinjaTemplateProcessor
|
||||
from superset.sql.parse import (
|
||||
_check_script_length,
|
||||
_count_weighted_table_references,
|
||||
_find_last_token_node,
|
||||
_get_select_trailing_child,
|
||||
BaseSQLStatement,
|
||||
count_referenced_tables,
|
||||
CTASMethod,
|
||||
@@ -1358,20 +1360,12 @@ LIMIT 100
|
||||
assert "increase timeout for large scans" in formatted[hint_end:]
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason=(
|
||||
"#38189 is not fully fixed: a `;`-terminated statement still hits "
|
||||
"the comment-relocation branch and corrupts the hint block. Only "
|
||||
"the no-semicolon form from the original repro was fixed."
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
def test_sqlscript_format_preserves_optimizer_hint_block_with_semicolon() -> None:
|
||||
"""
|
||||
Same as `test_sqlscript_format_preserves_optimizer_hint_block`, but with
|
||||
a terminating `;` on the statement -- this still reproduces #38189: the
|
||||
trailing `--` comment gets injected inside the `/*+ SET_VAR(...) */`
|
||||
hint block, corrupting it for StarRocks/MySQL-style engines.
|
||||
a terminating `;` on the statement -- verifies #38189 fix so that trailing
|
||||
`--` comments land after the statement rather than injected into the
|
||||
`/*+ SET_VAR(...) */` hint block for StarRocks/MySQL-style engines.
|
||||
"""
|
||||
sql = """SELECT /*+ SET_VAR(query_timeout = 3000) */ col1, col2
|
||||
FROM my_table
|
||||
@@ -1388,6 +1382,61 @@ LIMIT 100;
|
||||
assert "increase timeout for large scans" in formatted[hint_end:]
|
||||
|
||||
|
||||
def test_sqlscript_format_preserves_optimizer_hint_with_cte_and_semicolon() -> None:
|
||||
"""
|
||||
Ensure optimizer hints with CTEs and trailing comments survive formatting intact.
|
||||
"""
|
||||
sql = """WITH cte AS (SELECT 1 AS id)
|
||||
SELECT /*+ SET_VAR(query_timeout = 3000) */ id
|
||||
FROM cte
|
||||
WHERE id = 1;
|
||||
|
||||
-- trailing explanation comment"""
|
||||
statement = SQLScript(sql, "mysql").statements[0]
|
||||
formatted = statement.format()
|
||||
|
||||
hint = "/*+ SET_VAR(query_timeout = 3000) */"
|
||||
assert hint in formatted
|
||||
assert "SET_VAR(query_timeout /*" not in formatted
|
||||
hint_end = formatted.index(hint) + len(hint)
|
||||
assert "trailing explanation comment" in formatted[hint_end:]
|
||||
|
||||
|
||||
def test_find_last_token_node_branches() -> None:
|
||||
"""
|
||||
Directly test all branches of _find_last_token_node and _get_select_trailing_child.
|
||||
"""
|
||||
# 1. Empty select returns None from _get_select_trailing_child
|
||||
# and falls back to node
|
||||
empty_select = exp.Select()
|
||||
assert _get_select_trailing_child(empty_select) is None
|
||||
assert _find_last_token_node(empty_select) is empty_select
|
||||
|
||||
# 2. Select with list clause vs single Expression clause
|
||||
select_with_exprs = exp.Select(expressions=[exp.Literal.number(1)])
|
||||
assert _get_select_trailing_child(select_with_exprs) == exp.Literal.number(1)
|
||||
|
||||
select_with_where = exp.Select(where=exp.Where(this=exp.Literal.number(2)))
|
||||
assert _get_select_trailing_child(select_with_where) == exp.Literal.number(2)
|
||||
|
||||
# 3. Node with hint or comments in args is skipped during child traversal
|
||||
col_with_comment = exp.Column(this="foo", comments=["my comment"])
|
||||
assert _find_last_token_node(col_with_comment) is not None
|
||||
|
||||
table_with_hint = exp.Table(
|
||||
this="bar", hint=exp.Hint(expressions=[exp.var("HINT")])
|
||||
)
|
||||
assert _find_last_token_node(table_with_hint) is not None
|
||||
|
||||
# 4. Non-select node with list of expressions
|
||||
tup = exp.Tuple(expressions=[exp.Literal.number(1), exp.Literal.number(2)])
|
||||
assert _find_last_token_node(tup) == exp.Literal.number(2)
|
||||
|
||||
# 5. Leaf node with no children returns itself
|
||||
lit = exp.Literal.number(42)
|
||||
assert _find_last_token_node(lit) is lit
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sql, engine, expected",
|
||||
[
|
||||
|
||||
@@ -483,6 +483,11 @@ def test_get_sql_results_oauth2(mocker: MockerFixture, app) -> None:
|
||||
mocker.patch("superset.daos.key_value.KeyValueDAO.delete_expired_entries")
|
||||
mocker.patch("superset.daos.key_value.KeyValueDAO.create_entry")
|
||||
mocker.patch("superset.db_engine_specs.base.db.session.commit")
|
||||
# handle_query_error() refreshes `query` from the DB to check for a
|
||||
# concurrently-committed STOPPED status before overwriting it with
|
||||
# FAILED; `query` here is a MagicMock, not a real persistent ORM
|
||||
# instance, so the real refresh() would error introspecting it.
|
||||
mocker.patch("superset.sql_lab.db.session.refresh", return_value=None)
|
||||
|
||||
g = mocker.patch("superset.db_engine_specs.base.g")
|
||||
g.user = mocker.MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user