mirror of
https://github.com/apache/superset.git
synced 2026-09-09 00:34:49 +00:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fed40796fd | ||
|
|
5a11cc2177 | ||
|
|
39f93eb63c | ||
|
|
cf5307d0c6 | ||
|
|
9d1bc6b2cc | ||
|
|
6a125bf774 | ||
|
|
43fde2fb07 | ||
|
|
2be2246a00 | ||
|
|
80a5f6b787 | ||
|
|
c373da1bb9 | ||
|
|
80ea36c852 | ||
|
|
6ea4e22785 | ||
|
|
fcb1e299ac | ||
|
|
f4dfb7f026 | ||
|
|
001834470b | ||
|
|
e5c7200551 | ||
|
|
cb2a56d16e | ||
|
|
e5ff6de790 |
@@ -12,6 +12,11 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
||||
validate-all-ghas:
|
||||
|
||||
@@ -2,6 +2,11 @@ name: "Pull Request Labeler"
|
||||
on:
|
||||
- pull_request_target
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
permissions:
|
||||
|
||||
@@ -8,6 +8,11 @@ on:
|
||||
# Possible values: https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request
|
||||
types: [opened, edited, reopened, synchronize]
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint-check:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
name: E2E
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
# Gated behind pre-commit: this workflow runs only after the "pre-commit
|
||||
# checks" workflow completes, and (via the job-level `if` below) only when
|
||||
# it succeeded. That keeps the expensive Cypress/Playwright runners from
|
||||
# spinning up while a PR still has formatting/lint/type errors that
|
||||
# pre-commit catches in a fraction of the time. pre-commit itself runs on
|
||||
# push (master/release) and pull_request, so this preserves coverage for
|
||||
# both event types.
|
||||
workflow_run:
|
||||
workflows: ["pre-commit checks"]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
use_dashboard:
|
||||
@@ -23,11 +27,46 @@ on:
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
# workflow_run has no PR number in context; key on the originating branch.
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
# The pre-commit gate: only proceed when pre-commit succeeded (or on a
|
||||
# manual dispatch). On failure this job is skipped, and every downstream
|
||||
# job (needs: changes) is skipped with it — no runners are provisioned.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
|
||||
# The shared change-detector action reads the live event context, which
|
||||
# under workflow_run points at the default branch. Call the script
|
||||
# directly instead, passing the originating event/SHA/PR via WF_RUN_*.
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WF_RUN_EVENT: ${{ github.event.workflow_run.event }}
|
||||
WF_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
WF_RUN_PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}
|
||||
run: python scripts/change_detector.py
|
||||
|
||||
cypress-matrix:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
# Somehow one test flakes on 24.04 for unknown reasons, this is the only GHA left on 22.04
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
@@ -40,9 +79,14 @@ jobs:
|
||||
# https://github.com/cypress-io/github-action/issues/48
|
||||
fail-fast: false
|
||||
matrix:
|
||||
parallel_id: [0, 1, 2, 3, 4, 5]
|
||||
parallel_id: [0, 1]
|
||||
browser: ["chrome"]
|
||||
app_root: ${{ github.event_name == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
app_root: ${{ github.event.workflow_run.event == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
# The /app/prefix variant (push events only) is smoke-tested on a single
|
||||
# shard rather than the full matrix, so exclude it from the other shards.
|
||||
exclude:
|
||||
- parallel_id: 1
|
||||
app_root: "/app/prefix"
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -67,13 +111,13 @@ jobs:
|
||||
steps:
|
||||
# -------------------------------------------------------
|
||||
# Conditional checkout based on context
|
||||
- name: Checkout for push or pull_request event
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
- name: Checkout (gated by pre-commit via workflow_run)
|
||||
if: github.event_name == 'workflow_run'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
- name: Checkout using ref (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
@@ -89,51 +133,38 @@ jobs:
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
- name: Setup postgres
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: testdata
|
||||
- name: Setup Node.js
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version-file: './superset-frontend/.nvmrc'
|
||||
- name: Install npm dependencies
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Install cypress
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: cypress-install
|
||||
- name: Run Cypress
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
||||
PARALLEL_ID: ${{ matrix.parallel_id }}
|
||||
PARALLELISM: 6
|
||||
PARALLELISM: 2
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
with:
|
||||
@@ -154,6 +185,8 @@ jobs:
|
||||
name: cypress-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}-${{ matrix.parallel_id }}--${{ steps.set-safe-app-root.outputs.safe_app_root }}
|
||||
|
||||
playwright-tests:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -162,7 +195,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: ["chromium"]
|
||||
app_root: ${{ github.event_name == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
app_root: ${{ github.event.workflow_run.event == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -185,13 +218,13 @@ jobs:
|
||||
steps:
|
||||
# -------------------------------------------------------
|
||||
# Conditional checkout based on context (same as Cypress workflow)
|
||||
- name: Checkout for push or pull_request event
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
- name: Checkout (gated by pre-commit via workflow_run)
|
||||
if: github.event_name == 'workflow_run'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
- name: Checkout using ref (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
@@ -207,51 +240,37 @@ jobs:
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
- name: Setup postgres
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright_testdata
|
||||
- name: Setup Node.js
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version-file: './superset-frontend/.nvmrc'
|
||||
- name: Install npm dependencies
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Build embedded SDK
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-embedded-sdk
|
||||
- name: Install Playwright
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright-install
|
||||
- name: Run Playwright (Required Tests)
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
@@ -273,3 +292,34 @@ jobs:
|
||||
${{ github.workspace }}/superset-frontend/playwright-results/
|
||||
${{ github.workspace }}/superset-frontend/test-results/
|
||||
name: playwright-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }}
|
||||
|
||||
# workflow_run runs don't attach their checks to the originating PR, so post
|
||||
# an aggregate commit status back onto the PR head SHA. Make THIS the required
|
||||
# status check in branch protection (in place of the individual E2E jobs).
|
||||
report-status:
|
||||
needs: [cypress-matrix, playwright-tests]
|
||||
if: always() && github.event_name == 'workflow_run'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
statuses: write
|
||||
steps:
|
||||
- name: Report aggregate E2E status to PR commit
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
// 'skipped' is acceptable: the change-detector legitimately skips
|
||||
// jobs when no relevant files changed. Only real failures fail.
|
||||
const results = [
|
||||
'${{ needs.cypress-matrix.result }}',
|
||||
'${{ needs.playwright-tests.result }}',
|
||||
];
|
||||
const ok = results.every((r) => r === 'success' || r === 'skipped');
|
||||
await github.rest.repos.createCommitStatus({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
sha: context.payload.workflow_run.head_sha,
|
||||
state: ok ? 'success' : 'failure',
|
||||
context: 'E2E / required',
|
||||
description: ok ? 'E2E passed (or skipped)' : 'E2E failed',
|
||||
target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
|
||||
});
|
||||
|
||||
@@ -23,9 +23,30 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# NOTE: Required Playwright tests are in superset-e2e.yml (E2E / playwright-tests)
|
||||
# This workflow contains only experimental tests that run in shadow mode
|
||||
playwright-tests-experimental:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
runs-on: ubuntu-22.04
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
@@ -80,58 +101,43 @@ jobs:
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
- name: Setup postgres
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright_testdata
|
||||
- name: Setup Node.js
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version-file: './superset-frontend/.nvmrc'
|
||||
- name: Install npm dependencies
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Build embedded SDK
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-embedded-sdk
|
||||
- name: Install Playwright
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright-install
|
||||
- name: Run Playwright (Experimental Tests)
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
with:
|
||||
run: playwright-run "${{ matrix.app_root }}" experimental/
|
||||
- name: Run Playwright (Embedded Tests)
|
||||
if: steps.check.outputs.python || steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
|
||||
@@ -14,7 +14,27 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-mysql:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -29,9 +49,6 @@ jobs:
|
||||
image: mysql:8.0
|
||||
# Authenticated pulls use our higher Docker Hub rate limit. Empty on
|
||||
# fork PRs (secrets unavailable) -> runner falls back to anonymous.
|
||||
credentials:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
ports:
|
||||
@@ -43,9 +60,6 @@ jobs:
|
||||
--health-retries=5
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
credentials:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
options: --entrypoint redis-server
|
||||
ports:
|
||||
- 16379:6379
|
||||
@@ -55,26 +69,17 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python
|
||||
- name: Setup MySQL
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-mysql
|
||||
- name: Start Celery worker
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Python integration tests (MySQL)
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
@@ -85,7 +90,6 @@ jobs:
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
- name: Generate database diagnostics for docs
|
||||
if: steps.check.outputs.python
|
||||
env:
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
@@ -108,13 +112,14 @@ jobs:
|
||||
print(f'Generated diagnostics for {len(docs)} databases')
|
||||
"
|
||||
- name: Upload database diagnostics artifact
|
||||
if: steps.check.outputs.python
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: database-diagnostics
|
||||
path: databases-diagnostics.json
|
||||
retention-days: 7
|
||||
test-postgres:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -129,9 +134,6 @@ jobs:
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
credentials:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -141,9 +143,6 @@ jobs:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
credentials:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
@@ -152,29 +151,20 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
setup-postgres
|
||||
- name: Start Celery worker
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Python integration tests (PostgreSQL)
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
@@ -186,6 +176,8 @@ jobs:
|
||||
slug: apache/superset
|
||||
|
||||
test-sqlite:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -200,9 +192,6 @@ jobs:
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
credentials:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
@@ -211,28 +200,19 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python
|
||||
- name: Install dependencies
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
# sqlite needs this working directory
|
||||
mkdir ${{ github.workspace }}/.temp
|
||||
- name: Start Celery worker
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Python integration tests (SQLite)
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
|
||||
@@ -15,7 +15,27 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-postgres-presto:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -54,28 +74,17 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python == 'true'
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
echo "${{ steps.check.outputs.python }}"
|
||||
setup-postgres
|
||||
run: setup-postgres
|
||||
- name: Start Celery worker
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Python unit tests (PostgreSQL)
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
|
||||
- name: Upload code coverage
|
||||
@@ -87,6 +96,8 @@ jobs:
|
||||
slug: apache/superset
|
||||
|
||||
test-postgres-hive:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -117,35 +128,23 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Create csv upload directory
|
||||
if: steps.check.outputs.python
|
||||
run: sudo mkdir -p /tmp/.superset/uploads
|
||||
- name: Give write access to the csv upload directory
|
||||
if: steps.check.outputs.python
|
||||
run: sudo chown -R $USER:$USER /tmp/.superset
|
||||
- name: Start hadoop and hive
|
||||
if: steps.check.outputs.python
|
||||
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Start Celery worker
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Python unit tests (PostgreSQL)
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
pip install -e .[hive]
|
||||
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
|
||||
|
||||
@@ -15,7 +15,27 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
unit-tests:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -30,25 +50,17 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outputs.python
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Python unit tests
|
||||
if: steps.check.outputs.python
|
||||
env:
|
||||
SUPERSET_TESTENV: true
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
run: |
|
||||
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear --maxfail=50
|
||||
- name: Python 100% coverage unit tests
|
||||
if: steps.check.outputs.python
|
||||
env:
|
||||
SUPERSET_TESTENV: true
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
|
||||
+2
-1
@@ -154,7 +154,7 @@ fastmcp = [
|
||||
]
|
||||
firebird = ["sqlalchemy-firebird>=0.7.0, <2.2"]
|
||||
firebolt = ["firebolt-sqlalchemy>=1.0.0, <2"]
|
||||
gevent = ["gevent>=23.9.1"]
|
||||
gevent = ["gevent>=26.4.0"]
|
||||
gsheets = ["shillelagh[gsheetsapi]>=1.4.4, <2"]
|
||||
hana = ["hdbcli==2.28.20", "sqlalchemy_hana==0.4.0"]
|
||||
hive = [
|
||||
@@ -456,6 +456,7 @@ authorized_licenses = [
|
||||
"isc license (iscl)",
|
||||
"isc license",
|
||||
"mit",
|
||||
"mit and psf-2.0",
|
||||
"mit-cmu",
|
||||
"mozilla public license 2.0 (mpl 2.0)",
|
||||
"osi approved",
|
||||
|
||||
@@ -161,7 +161,7 @@ geopy==2.4.1
|
||||
# via apache-superset (pyproject.toml)
|
||||
google-auth==2.43.0
|
||||
# via shillelagh
|
||||
greenlet==3.1.1
|
||||
greenlet==3.5.0
|
||||
# via
|
||||
# apache-superset (pyproject.toml)
|
||||
# shillelagh
|
||||
|
||||
@@ -331,7 +331,7 @@ geopy==2.4.1
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
gevent==24.2.1
|
||||
gevent==26.4.0
|
||||
# via apache-superset
|
||||
google-api-core==2.23.0
|
||||
# via
|
||||
@@ -373,7 +373,7 @@ googleapis-common-protos==1.66.0
|
||||
# via
|
||||
# google-api-core
|
||||
# grpcio-status
|
||||
greenlet==3.1.1
|
||||
greenlet==3.5.0
|
||||
# via
|
||||
# -c requirements/base-constraint.txt
|
||||
# apache-superset
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
@@ -109,6 +109,37 @@ def is_int(s: str) -> bool:
|
||||
return bool(re.match(r"^-?\d+$", s))
|
||||
|
||||
|
||||
def resolve_workflow_run_files(repo: str, sha: str) -> Optional[List[str]]:
|
||||
"""Resolve changed files for a workflow_run-triggered run.
|
||||
|
||||
When a workflow is gated behind another (e.g. running only after
|
||||
pre-commit succeeds), GitHub re-dispatches it as a `workflow_run` event
|
||||
whose context points at the default branch rather than the originating
|
||||
diff. Recover the original event and head SHA from the workflow_run
|
||||
payload, exposed via the WF_RUN_* env vars. Returns ``None`` (meaning
|
||||
"assume all changed") when the diff can't be resolved.
|
||||
"""
|
||||
original_event = os.getenv("WF_RUN_EVENT") or "push"
|
||||
print("ORIGINAL_EVENT", original_event)
|
||||
if original_event == "pull_request":
|
||||
pr_number = os.getenv("WF_RUN_PR_NUMBER", "")
|
||||
if not is_int(pr_number):
|
||||
# Fork PRs don't populate workflow_run.pull_requests, so we can't
|
||||
# resolve the diff -> assume all changed (run everything).
|
||||
print("workflow_run without PR context, assuming all changed")
|
||||
return None
|
||||
files = fetch_changed_files_pr(repo, pr_number)
|
||||
print("PR files:")
|
||||
print_files(files)
|
||||
return files
|
||||
|
||||
head_sha = os.getenv("WF_RUN_HEAD_SHA") or sha
|
||||
files = fetch_changed_files_push(repo, head_sha)
|
||||
print("Files touched since previous commit:")
|
||||
print_files(files)
|
||||
return files
|
||||
|
||||
|
||||
def main(event_type: str, sha: str, repo: str) -> None:
|
||||
"""Main function to check for file changes based on event context."""
|
||||
print("SHA:", sha)
|
||||
@@ -126,6 +157,9 @@ def main(event_type: str, sha: str, repo: str) -> None:
|
||||
print("Files touched since previous commit:")
|
||||
print_files(files)
|
||||
|
||||
elif event_type == "workflow_run":
|
||||
files = resolve_workflow_run_files(repo, sha)
|
||||
|
||||
elif event_type in ("workflow_dispatch", "schedule"):
|
||||
# Manual or cron-triggered runs aren't tied to a specific diff, so
|
||||
# treat every group as changed. `files = None` makes the loop below
|
||||
|
||||
@@ -18,14 +18,31 @@
|
||||
"""
|
||||
Check that source-code changes don't cause translation regressions.
|
||||
|
||||
What counts as a regression
|
||||
---------------------------
|
||||
A regression is an *existing translation that a source change invalidated* —
|
||||
i.e. a string was renamed/reworded so its committed translation no longer
|
||||
applies. ``babel_update.sh`` (``pybabel update --ignore-obsolete``) surfaces
|
||||
exactly these as **newly fuzzy** entries: the old translation is fuzzy-matched
|
||||
onto the new ``msgid`` and flagged ``#, fuzzy``.
|
||||
|
||||
Crucially, *deleting* a translatable string is **not** a regression. With
|
||||
``--ignore-obsolete`` a removed string is dropped from the catalogs entirely;
|
||||
no fuzzy entry is created. So a PR that intentionally removes a string (e.g. a
|
||||
security fix that stops rendering a value) legitimately lowers the translated
|
||||
count without introducing any fuzzies, and must not be flagged. We therefore
|
||||
key the check on the **increase in fuzzy entries**, not on a drop in the
|
||||
translated count (a drop happens identically for a benign deletion and a real
|
||||
rename, so it cannot distinguish the two).
|
||||
|
||||
Usage
|
||||
-----
|
||||
Count non-fuzzy translated entries in all .po files and write JSON to stdout:
|
||||
Count translated + fuzzy entries in all .po files and write JSON to stdout:
|
||||
|
||||
python check_translation_regression.py --count
|
||||
|
||||
Compare the current .po state against a previously-recorded baseline and fail
|
||||
if any language lost translations:
|
||||
if a source change invalidated existing translations (new fuzzies):
|
||||
|
||||
python check_translation_regression.py --compare /path/to/before.json
|
||||
|
||||
@@ -50,8 +67,8 @@ Typical CI workflow
|
||||
|
||||
Running babel_update on the base branch first isolates regressions caused by
|
||||
the PR's source diff from any pre-existing drift on the base branch, while the
|
||||
PR worktree run still allows committed .po updates to restore lost
|
||||
translations.
|
||||
PR worktree run still allows committed .po updates to resolve the fuzzies (and
|
||||
thus clear the regression) before merging.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -71,8 +88,13 @@ DEFAULT_TRANSLATIONS_DIR = (
|
||||
SKIP_LANGS = {"en"}
|
||||
|
||||
|
||||
def count_translated(po_file: Path) -> int:
|
||||
"""Return the number of non-fuzzy translated messages in a .po file.
|
||||
def count_stats(po_file: Path) -> dict[str, int]:
|
||||
"""Return ``{"translated": int, "fuzzy": int}`` for a .po file.
|
||||
|
||||
``translated`` is the number of non-fuzzy translated messages; ``fuzzy`` is
|
||||
the number of fuzzy translations. The fuzzy count is what the regression
|
||||
check keys on — a source rename invalidates an existing translation by
|
||||
making it fuzzy, whereas a deletion simply drops it (``--ignore-obsolete``).
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: if ``msgfmt`` fails (e.g. malformed
|
||||
@@ -90,29 +112,50 @@ def count_translated(po_file: Path) -> int:
|
||||
check=True,
|
||||
)
|
||||
# stderr: "123 translated messages, 4 fuzzy translations, 56 untranslated messages."
|
||||
match = re.search(r"(\d+) translated message", result.stderr)
|
||||
if not match:
|
||||
# The fuzzy and untranslated clauses are omitted by msgfmt when they are 0.
|
||||
translated_match = re.search(r"(\d+) translated message", result.stderr)
|
||||
if not translated_match:
|
||||
raise RuntimeError(
|
||||
f"Could not parse msgfmt --statistics output for {po_file}: "
|
||||
f"{result.stderr!r}"
|
||||
)
|
||||
return int(match.group(1))
|
||||
fuzzy_match = re.search(r"(\d+) fuzzy translation", result.stderr)
|
||||
return {
|
||||
"translated": int(translated_match.group(1)),
|
||||
"fuzzy": int(fuzzy_match.group(1)) if fuzzy_match else 0,
|
||||
}
|
||||
|
||||
|
||||
def get_counts(translations_dir: Path) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
def get_counts(
|
||||
translations_dir: Path,
|
||||
failures: Optional[set[str]] = None,
|
||||
) -> dict[str, dict[str, int]]:
|
||||
"""Count translated/fuzzy entries for every ``.po`` file in a directory.
|
||||
|
||||
If ``failures`` is provided, the name of each language whose ``.po`` file
|
||||
is present on disk but could not be counted (msgfmt non-zero exit, or
|
||||
unparseable output) is added to it. Such a language is deliberately absent
|
||||
from the returned mapping — but, unlike a language whose catalog was simply
|
||||
deleted, it must not be mistaken for an intentional removal: a caller that
|
||||
cares about the distinction (see :func:`cmd_compare`) can inspect
|
||||
``failures`` and treat it as a hard error.
|
||||
"""
|
||||
counts: dict[str, dict[str, int]] = {}
|
||||
for po_file in sorted(translations_dir.glob("*/LC_MESSAGES/messages.po")):
|
||||
lang = po_file.parent.parent.name
|
||||
if lang in SKIP_LANGS:
|
||||
continue
|
||||
try:
|
||||
counts[lang] = count_translated(po_file)
|
||||
counts[lang] = count_stats(po_file)
|
||||
except (subprocess.CalledProcessError, RuntimeError) as exc:
|
||||
# A malformed .po file (msgfmt non-zero exit, or stderr we
|
||||
# can't parse) is a real problem worth seeing, but it shouldn't
|
||||
# take the whole regression check down with it — that would
|
||||
# hide every other language's status. Skip and warn instead;
|
||||
# the missing lang will not appear in the comparison output.
|
||||
# hide every other language's status. Skip and warn here; the
|
||||
# caller is told which langs failed via ``failures`` so it can
|
||||
# decide whether a present-but-uncountable catalog is fatal.
|
||||
if failures is not None:
|
||||
failures.add(lang)
|
||||
print(
|
||||
f"WARNING: skipping {lang} — {po_file} could not be counted: {exc}",
|
||||
file=sys.stderr,
|
||||
@@ -120,18 +163,42 @@ def get_counts(translations_dir: Path) -> dict[str, int]:
|
||||
return counts
|
||||
|
||||
|
||||
def _normalize(entry: object) -> dict[str, int]:
|
||||
"""Coerce a baseline entry into ``{"translated", "fuzzy"}``.
|
||||
|
||||
Tolerates the legacy baseline format where each language mapped directly to
|
||||
an integer translated count (no fuzzy data); such entries contribute a
|
||||
fuzzy baseline of 0.
|
||||
"""
|
||||
if isinstance(entry, dict):
|
||||
return {
|
||||
"translated": int(entry.get("translated", 0)),
|
||||
"fuzzy": int(entry.get("fuzzy", 0)),
|
||||
}
|
||||
if isinstance(entry, int):
|
||||
return {"translated": entry, "fuzzy": 0}
|
||||
raise TypeError(f"Unsupported baseline entry: {entry!r}")
|
||||
|
||||
|
||||
def build_regression_report(regressions: list[tuple[str, int, int]]) -> str:
|
||||
"""Build a markdown report for posting as a PR comment."""
|
||||
"""Build a markdown report for posting as a PR comment.
|
||||
|
||||
Each regression tuple is ``(lang, before_fuzzy, after_fuzzy)``.
|
||||
"""
|
||||
rows = "\n".join(
|
||||
f"| `{lang}` | {b} | {a} | -{b - a} |" for lang, b, a in regressions
|
||||
f"| `{lang}` | {b} | {a} | +{a - b} |" for lang, b, a in regressions
|
||||
)
|
||||
affected = ", ".join(f"`{lang}`" for lang, _, _ in regressions)
|
||||
return (
|
||||
"## ⚠️ Translation Regression Detected\n\n"
|
||||
f"This PR causes existing translations to become fuzzy or be removed "
|
||||
f"in {affected}. Please fix the affected `.po` files before merging.\n\n"
|
||||
"| Language | Before | After | Lost |\n"
|
||||
"|----------|-------:|------:|-----:|\n"
|
||||
f"A source change in this PR renamed or reworded strings, invalidating "
|
||||
f"existing translations (they are now `#, fuzzy`) in {affected}. Please "
|
||||
f"resolve the affected `.po` files before merging.\n\n"
|
||||
"_Note: intentionally **deleting** a translatable string is not a "
|
||||
"regression and is not flagged here — only translations invalidated by "
|
||||
"a renamed/reworded source string are._\n\n"
|
||||
"| Language | Fuzzy before | Fuzzy after | New |\n"
|
||||
"|----------|-------------:|------------:|----:|\n"
|
||||
f"{rows}\n\n"
|
||||
"### How to fix\n\n"
|
||||
"**1. Install dependencies** (if not already set up):\n\n"
|
||||
@@ -169,26 +236,49 @@ def cmd_compare(
|
||||
report_path: Optional[str] = None,
|
||||
) -> None:
|
||||
with open(before_path) as f:
|
||||
before: dict[str, int] = json.load(f)
|
||||
before_raw: dict[str, object] = json.load(f)
|
||||
before = {lang: _normalize(entry) for lang, entry in before_raw.items()}
|
||||
|
||||
after = get_counts(translations_dir)
|
||||
failures: set[str] = set()
|
||||
after = get_counts(translations_dir, failures=failures)
|
||||
|
||||
# A baseline language whose catalog is *missing* from `after` is fine —
|
||||
# that's an intentional catalog deletion (handled below like any other
|
||||
# deletion). But a language whose .po file is still present yet could not
|
||||
# be counted (msgfmt failed / output unparseable) is a hard error: leaving
|
||||
# it out silently would let a corrupt catalog pass as "no regression".
|
||||
broken = sorted(lang for lang in failures if lang in before)
|
||||
if broken:
|
||||
print("Translation check failed!\n")
|
||||
for lang in broken:
|
||||
print(f" {lang}: catalog present but could not be counted (msgfmt error)")
|
||||
print(
|
||||
"\nFix the malformed .po file(s) above before merging — a catalog "
|
||||
"that cannot be parsed must not be silently dropped."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# A regression is an *increase* in fuzzy entries: the PR's source diff
|
||||
# renamed/reworded strings, leaving their committed translations stranded.
|
||||
# A plain drop in the translated count is NOT used — deleting a string
|
||||
# lowers it identically to a rename but is a legitimate change, and with
|
||||
# `pybabel update --ignore-obsolete` a deletion creates no fuzzy entry.
|
||||
regressions: list[tuple[str, int, int]] = []
|
||||
for lang, before_count in sorted(before.items()):
|
||||
after_count = after.get(lang, 0)
|
||||
if after_count < before_count:
|
||||
regressions.append((lang, before_count, after_count))
|
||||
for lang, before_stats in sorted(before.items()):
|
||||
after_stats = after.get(lang, {"translated": 0, "fuzzy": 0})
|
||||
if after_stats["fuzzy"] > before_stats["fuzzy"]:
|
||||
regressions.append((lang, before_stats["fuzzy"], after_stats["fuzzy"]))
|
||||
|
||||
if regressions:
|
||||
print("Translation regression detected!\n")
|
||||
for lang, b, a in regressions:
|
||||
lost = b - a
|
||||
print(f" {lang}: {b} -> {a} (-{lost} string(s) became fuzzy or removed)")
|
||||
print(
|
||||
f" {lang}: {a - b} translation(s) invalidated "
|
||||
f"(fuzzy {b} -> {a}) by a renamed/reworded source string"
|
||||
)
|
||||
print(
|
||||
"\nStrings renamed or deleted by this PR invalidated existing translations."
|
||||
)
|
||||
print(
|
||||
"Update the affected .po files to restore the lost entries before merging."
|
||||
"\nResolve the newly-fuzzy entries in the affected .po files "
|
||||
"before merging."
|
||||
)
|
||||
if report_path:
|
||||
Path(report_path).write_text(
|
||||
@@ -199,15 +289,15 @@ def cmd_compare(
|
||||
# All good — print a summary so it's easy to read in CI logs.
|
||||
print("No translation regressions.\n")
|
||||
for lang in sorted(after):
|
||||
b = before.get(lang, 0)
|
||||
a = after[lang]
|
||||
if a > b:
|
||||
delta = f"+{a - b}"
|
||||
elif a == b:
|
||||
delta = "no change"
|
||||
else:
|
||||
delta = f"-{b - a}"
|
||||
print(f" {lang}: {b} -> {a} ({delta})")
|
||||
before_stats = before.get(lang, {"translated": 0, "fuzzy": 0})
|
||||
after_stats = after[lang]
|
||||
t_delta = after_stats["translated"] - before_stats["translated"]
|
||||
f_delta = after_stats["fuzzy"] - before_stats["fuzzy"]
|
||||
print(
|
||||
f" {lang}: translated {before_stats['translated']} -> "
|
||||
f"{after_stats['translated']} ({t_delta:+d}), fuzzy "
|
||||
f"{before_stats['fuzzy']} -> {after_stats['fuzzy']} ({f_delta:+d})"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
|
||||
import { getTimeFormatter } from '@superset-ui/core';
|
||||
|
||||
// Cal-Heatmap provides local timestamps. We subtract the offset so that utcFormat displays the correct local date.
|
||||
// Cal-Heatmap provides local timestamps (UTC shifted by the browser's timezone
|
||||
// offset). We subtract that offset so the formatter displays the correct UTC
|
||||
// date regardless of the browser's timezone.
|
||||
export const getFormattedUTCTime = (
|
||||
ts: number | string,
|
||||
timeFormat?: string,
|
||||
|
||||
+18
-9
@@ -299,18 +299,23 @@ var CalHeatMap = function () {
|
||||
// Takes the fetched "data" object as argument, must return a json object
|
||||
// formatted like {timestamp:count, timestamp2:count2},
|
||||
afterLoadData: function (timestamps) {
|
||||
// See https://github.com/wa0x6e/cal-heatmap/issues/126#issuecomment-373301803
|
||||
const stdTimezoneOffset = date => {
|
||||
const jan = new Date(date.getFullYear(), 0, 1);
|
||||
const jul = new Date(date.getFullYear(), 6, 1);
|
||||
return Math.max(jan.getTimezoneOffset(), jul.getTimezoneOffset());
|
||||
};
|
||||
const offset = stdTimezoneOffset(new Date()) * 60;
|
||||
// Use the DST-aware timezone offset for each individual timestamp so that
|
||||
// every data point is shifted by its own local offset (not a fixed
|
||||
// standard-time offset). This prevents data from landing in phantom hours
|
||||
// during DST transitions and keeps the offset consistent with what
|
||||
// getFormattedUTCTime undoes when formatting the tooltip.
|
||||
//
|
||||
// Around DST transitions two distinct UTC timestamps can shift to the
|
||||
// same adjusted key (e.g. the "spring forward" hour that doesn't exist
|
||||
// locally). Accumulate values on collision so no datapoints are silently
|
||||
// dropped in hourly/minutely views.
|
||||
let results = {};
|
||||
for (let timestamp in timestamps) {
|
||||
const value = timestamps[timestamp];
|
||||
timestamp = parseInt(timestamp, 10);
|
||||
results[timestamp + offset] = value;
|
||||
const ts = parseInt(timestamp, 10);
|
||||
const offset = new Date(ts * 1000).getTimezoneOffset() * 60;
|
||||
const adjustedTs = ts + offset;
|
||||
results[adjustedTs] = (results[adjustedTs] || 0) + value;
|
||||
}
|
||||
return results;
|
||||
},
|
||||
@@ -4005,6 +4010,10 @@ function mergeRecursive(obj1, obj2) {
|
||||
|
||||
/*jshint forin:false */
|
||||
for (var p in obj2) {
|
||||
// Skip keys that could pollute the object prototype.
|
||||
if (p === '__proto__' || p === 'constructor' || p === 'prototype') {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// Property in destination object set; update its value.
|
||||
if (obj2[p].constructor === Object) {
|
||||
|
||||
@@ -19,78 +19,71 @@
|
||||
|
||||
import { getFormattedUTCTime, convertUTCTimestampToLocal } from '../src/utils';
|
||||
|
||||
describe('getFormattedUTCTime', () => {
|
||||
test('formats local timestamp for display as UTC date', () => {
|
||||
const utcTimestamp = 1420070400000; // 2015-01-01 00:00:00 UTC
|
||||
const localTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const formattedTime = getFormattedUTCTime(
|
||||
localTimestamp,
|
||||
'%Y-%m-%d %H:%M:%S',
|
||||
);
|
||||
test('getFormattedUTCTime formats local timestamp for display as UTC date', () => {
|
||||
const utcTimestamp = 1420070400000; // 2015-01-01 00:00:00 UTC
|
||||
const localTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
// Cal-Heatmap's afterLoadData adjusts timestamps similarly, so
|
||||
// getFormattedUTCTime receives already-adjusted timestamps and
|
||||
// formats them directly. The date component should be correct.
|
||||
const formattedTime = getFormattedUTCTime(localTimestamp, '%Y-%m-%d');
|
||||
|
||||
expect(formattedTime).toEqual('2015-01-01 00:00:00');
|
||||
});
|
||||
expect(formattedTime).toEqual('2015-01-01');
|
||||
});
|
||||
|
||||
describe('convertUTCTimestampToLocal', () => {
|
||||
test('adjusts timestamp so local Date shows UTC date', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
const adjustedTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const adjustedDate = new Date(adjustedTimestamp);
|
||||
test('convertUTCTimestampToLocal adjusts timestamp so local Date shows UTC date', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
const adjustedTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const adjustedDate = new Date(adjustedTimestamp);
|
||||
|
||||
expect(adjustedDate.getFullYear()).toEqual(2024);
|
||||
expect(adjustedDate.getMonth()).toEqual(0);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('handles month boundaries', () => {
|
||||
const utcTimestamp = 1706745600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(utcTimestamp));
|
||||
|
||||
expect(adjustedDate.getFullYear()).toEqual(2024);
|
||||
expect(adjustedDate.getMonth()).toEqual(1);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('handles year boundaries', () => {
|
||||
const utcTimestamp = 1735689600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(utcTimestamp));
|
||||
|
||||
expect(adjustedDate.getFullYear()).toEqual(2025);
|
||||
expect(adjustedDate.getMonth()).toEqual(0);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('adds timezone offset to timestamp', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
const adjustedTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const expectedOffset =
|
||||
new Date(utcTimestamp).getTimezoneOffset() * 60 * 1000;
|
||||
|
||||
expect(adjustedTimestamp - utcTimestamp).toEqual(expectedOffset);
|
||||
});
|
||||
expect(adjustedDate.getFullYear()).toEqual(2024);
|
||||
expect(adjustedDate.getMonth()).toEqual(0);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
describe('integration', () => {
|
||||
test('fixes timezone bug for CalHeatMap', () => {
|
||||
const febFirst2024UTC = 1706745600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(febFirst2024UTC));
|
||||
test('convertUTCTimestampToLocal handles month boundaries', () => {
|
||||
const utcTimestamp = 1706745600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(utcTimestamp));
|
||||
|
||||
expect(adjustedDate.getMonth()).toEqual(1);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('both functions work together to display dates correctly', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
|
||||
// convertUTCTimestampToLocal adjusts UTC for Cal-Heatmap (which interprets as local)
|
||||
const localTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const calHeatmapDate = new Date(localTimestamp);
|
||||
expect(calHeatmapDate.getMonth()).toEqual(0);
|
||||
expect(calHeatmapDate.getDate()).toEqual(1);
|
||||
|
||||
// getFormattedUTCTime receives LOCAL timestamp (from Cal-Heatmap) and formats it
|
||||
const formattedTime = getFormattedUTCTime(localTimestamp, '%Y-%m-%d');
|
||||
expect(formattedTime).toContain('2024-01-01');
|
||||
});
|
||||
expect(adjustedDate.getFullYear()).toEqual(2024);
|
||||
expect(adjustedDate.getMonth()).toEqual(1);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('convertUTCTimestampToLocal handles year boundaries', () => {
|
||||
const utcTimestamp = 1735689600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(utcTimestamp));
|
||||
|
||||
expect(adjustedDate.getFullYear()).toEqual(2025);
|
||||
expect(adjustedDate.getMonth()).toEqual(0);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('convertUTCTimestampToLocal adds timezone offset to timestamp', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
const adjustedTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const expectedOffset = new Date(utcTimestamp).getTimezoneOffset() * 60 * 1000;
|
||||
|
||||
expect(adjustedTimestamp - utcTimestamp).toEqual(expectedOffset);
|
||||
});
|
||||
|
||||
test('convertUTCTimestampToLocal fixes timezone bug for CalHeatMap', () => {
|
||||
const febFirst2024UTC = 1706745600000;
|
||||
const adjustedDate = new Date(convertUTCTimestampToLocal(febFirst2024UTC));
|
||||
|
||||
expect(adjustedDate.getMonth()).toEqual(1);
|
||||
expect(adjustedDate.getDate()).toEqual(1);
|
||||
});
|
||||
|
||||
test('convertUTCTimestampToLocal and getFormattedUTCTime work together to display dates correctly', () => {
|
||||
const utcTimestamp = 1704067200000;
|
||||
|
||||
// convertUTCTimestampToLocal adjusts UTC for Cal-Heatmap (which interprets as local)
|
||||
const localTimestamp = convertUTCTimestampToLocal(utcTimestamp);
|
||||
const calHeatmapDate = new Date(localTimestamp);
|
||||
expect(calHeatmapDate.getMonth()).toEqual(0);
|
||||
expect(calHeatmapDate.getDate()).toEqual(1);
|
||||
|
||||
// getFormattedUTCTime receives LOCAL timestamp (from Cal-Heatmap) and formats it
|
||||
const formattedTime = getFormattedUTCTime(localTimestamp, '%Y-%m-%d');
|
||||
expect(formattedTime).toContain('2024-01-01');
|
||||
});
|
||||
|
||||
@@ -275,29 +275,29 @@ export function wrapTooltip(chart) {
|
||||
});
|
||||
}
|
||||
|
||||
// Builds the sanitized HTML for an annotation layer's tooltip. Title and
|
||||
// description values come from the annotation data source, so the output is
|
||||
// run through dompurify before being inserted into the DOM by d3-tip.
|
||||
export function generateAnnotationTooltipContent(layer, d) {
|
||||
const title =
|
||||
d[layer.titleColumn] && d[layer.titleColumn].length > 0
|
||||
? `${d[layer.titleColumn]} - ${layer.name}`
|
||||
: layer.name;
|
||||
const body = Array.isArray(layer.descriptionColumns)
|
||||
? layer.descriptionColumns.map(c => d[c])
|
||||
: Object.values(d);
|
||||
|
||||
return dompurify.sanitize(
|
||||
`<div><strong>${title}</strong></div><br/><div>${body.join(', ')}</div>`,
|
||||
);
|
||||
}
|
||||
|
||||
export function tipFactory(layer) {
|
||||
return d3tip()
|
||||
.attr('class', `d3-tip ${layer.annotationTipClass || ''}`)
|
||||
.direction('n')
|
||||
.offset([-5, 0])
|
||||
.html(d => {
|
||||
if (!d) {
|
||||
return '';
|
||||
}
|
||||
const rawTitle =
|
||||
d[layer.titleColumn] && d[layer.titleColumn].length > 0
|
||||
? `${d[layer.titleColumn]} - ${layer.name}`
|
||||
: layer.name;
|
||||
const rawBody = Array.isArray(layer.descriptionColumns)
|
||||
? layer.descriptionColumns.map(c => d[c])
|
||||
: Object.values(d);
|
||||
|
||||
return dompurify.sanitize(
|
||||
`<div><strong>${rawTitle}</strong></div><br/><div>${rawBody.join(
|
||||
', ',
|
||||
)}</div>`,
|
||||
);
|
||||
});
|
||||
.html(d => (d ? generateAnnotationTooltipContent(layer, d) : ''));
|
||||
}
|
||||
|
||||
export function getMaxLabelSize(svg, axisClass) {
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
|
||||
import {
|
||||
computeYDomain,
|
||||
generateAnnotationTooltipContent,
|
||||
generateBubbleTooltipContent,
|
||||
generateMultiLineTooltipContent,
|
||||
getTimeOrNumberFormatter,
|
||||
@@ -125,6 +126,42 @@ describe('nvd3/utils', () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe('generateMultiLineTooltipContent()', () => {
|
||||
const identity = (value: any) => value;
|
||||
|
||||
test('renders the series key in the tooltip markup', () => {
|
||||
const tooltip = generateMultiLineTooltipContent(
|
||||
{
|
||||
value: 'x-value',
|
||||
series: [{ key: 'Region A', color: '#fff', value: 1 }],
|
||||
},
|
||||
identity,
|
||||
[identity],
|
||||
);
|
||||
expect(tooltip).toContain('Region A');
|
||||
});
|
||||
|
||||
test('strips a script payload from a malicious series key', () => {
|
||||
const tooltip = generateMultiLineTooltipContent(
|
||||
{
|
||||
value: 'x-value',
|
||||
series: [
|
||||
{
|
||||
key: '<img src=x onerror="alert(1)">',
|
||||
color: '#fff',
|
||||
value: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
identity,
|
||||
[identity],
|
||||
);
|
||||
// DOMPurify removes the event handler that would execute on render.
|
||||
expect(tooltip).not.toContain('onerror');
|
||||
expect(tooltip).not.toContain('alert(1)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTimeOrNumberFormatter(format)', () => {
|
||||
test('is a function', () => {
|
||||
expect(typeof getTimeOrNumberFormatter).toBe('function');
|
||||
@@ -276,4 +313,46 @@ describe('nvd3/utils', () => {
|
||||
expect(html).toContain('payload');
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateAnnotationTooltipContent()', () => {
|
||||
const layer = {
|
||||
name: 'My annotations',
|
||||
titleColumn: 'title',
|
||||
descriptionColumns: ['description'],
|
||||
};
|
||||
|
||||
test('renders the annotation title and description', () => {
|
||||
const html = generateAnnotationTooltipContent(layer, {
|
||||
title: 'Release',
|
||||
description: 'Shipped v1',
|
||||
});
|
||||
expect(html).toContain('Release - My annotations');
|
||||
expect(html).toContain('Shipped v1');
|
||||
});
|
||||
|
||||
test('falls back to the layer name when the title column is empty', () => {
|
||||
const html = generateAnnotationTooltipContent(layer, {
|
||||
title: '',
|
||||
description: 'Shipped v1',
|
||||
});
|
||||
expect(html).toContain('My annotations');
|
||||
});
|
||||
|
||||
test('strips an event-handler payload from the title column', () => {
|
||||
const html = generateAnnotationTooltipContent(layer, {
|
||||
title: '<img src=x onerror="alert(1)">',
|
||||
description: 'ok',
|
||||
});
|
||||
expect(html).not.toContain('onerror');
|
||||
expect(html).not.toContain('alert(1)');
|
||||
});
|
||||
|
||||
test('strips a script payload from a description column', () => {
|
||||
const html = generateAnnotationTooltipContent(layer, {
|
||||
title: 'Release',
|
||||
description: '<script>alert(document.cookie)</script>',
|
||||
});
|
||||
expect(html).not.toContain('<script>');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+1
-1
@@ -1193,7 +1193,7 @@ HTML_SANITIZATION_SCHEMA_EXTENSIONS: dict[str, Any] = {}
|
||||
# than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for
|
||||
# next available socket. PR #5039 added domain sharding for Superset,
|
||||
# and this feature can be enabled by configuration only (by default Superset
|
||||
# doesn't allow cross-domain request). This feature is deprecated, annd will be removed
|
||||
# doesn't allow cross-domain request). This feature is deprecated, and will be removed
|
||||
# in the next major version of Superset, as enabling HTTP2 will serve the same goals.
|
||||
SUPERSET_WEBSERVER_DOMAINS = None # deprecated
|
||||
|
||||
|
||||
+30
-7
@@ -80,13 +80,24 @@ class ColumnOperatorEnum(str, Enum):
|
||||
return op_func(column, value)
|
||||
|
||||
|
||||
def _escape_like(value: str) -> str:
|
||||
"""Escape LIKE/ILIKE wildcards to prevent wildcard injection."""
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
# Define operator_map as a module-level dict after the enum is defined
|
||||
operator_map: Dict[ColumnOperatorEnum, Any] = {
|
||||
ColumnOperatorEnum.eq: lambda col, val: col == val,
|
||||
ColumnOperatorEnum.ne: lambda col, val: col != val,
|
||||
ColumnOperatorEnum.sw: lambda col, val: col.like(f"{val}%"),
|
||||
ColumnOperatorEnum.ew: lambda col, val: col.like(f"%{val}"),
|
||||
ColumnOperatorEnum.ct: lambda col, val: col.ilike(f"%{val}%"),
|
||||
ColumnOperatorEnum.sw: lambda col, val: col.like(
|
||||
f"{_escape_like(val)}%", escape="\\"
|
||||
),
|
||||
ColumnOperatorEnum.ew: lambda col, val: col.like(
|
||||
f"%{_escape_like(val)}", escape="\\"
|
||||
),
|
||||
ColumnOperatorEnum.ct: lambda col, val: col.ilike(
|
||||
f"%{_escape_like(val)}%", escape="\\"
|
||||
),
|
||||
ColumnOperatorEnum.in_: lambda col, val: col.in_(
|
||||
val if isinstance(val, (list, tuple)) else [val]
|
||||
),
|
||||
@@ -97,8 +108,12 @@ operator_map: Dict[ColumnOperatorEnum, Any] = {
|
||||
ColumnOperatorEnum.gte: lambda col, val: col >= val,
|
||||
ColumnOperatorEnum.lt: lambda col, val: col < val,
|
||||
ColumnOperatorEnum.lte: lambda col, val: col <= val,
|
||||
ColumnOperatorEnum.like: lambda col, val: col.like(f"%{val}%"),
|
||||
ColumnOperatorEnum.ilike: lambda col, val: col.ilike(f"%{val}%"),
|
||||
ColumnOperatorEnum.like: lambda col, val: col.like(
|
||||
f"%{_escape_like(val)}%", escape="\\"
|
||||
),
|
||||
ColumnOperatorEnum.ilike: lambda col, val: col.ilike(
|
||||
f"%{_escape_like(val)}%", escape="\\"
|
||||
),
|
||||
ColumnOperatorEnum.is_null: lambda col, _: col.is_(None),
|
||||
ColumnOperatorEnum.is_not_null: lambda col, _: col.isnot(None),
|
||||
}
|
||||
@@ -657,7 +672,11 @@ class BaseDAO(CoreBaseDAO[T], Generic[T]):
|
||||
for column_name in search_columns:
|
||||
if hasattr(cls.model_cls, column_name):
|
||||
column = getattr(cls.model_cls, column_name)
|
||||
search_filters.append(cast(column, Text).ilike(f"%{search}%"))
|
||||
search_filters.append(
|
||||
cast(column, Text).ilike(
|
||||
f"%{_escape_like(search)}%", escape="\\"
|
||||
)
|
||||
)
|
||||
if search_filters:
|
||||
query = query.filter(or_(*search_filters))
|
||||
if custom_filters:
|
||||
@@ -724,7 +743,11 @@ class BaseDAO(CoreBaseDAO[T], Generic[T]):
|
||||
for column_name in search_columns:
|
||||
if hasattr(cls.model_cls, column_name):
|
||||
column = getattr(cls.model_cls, column_name)
|
||||
search_filters.append(cast(column, Text).ilike(f"%{search}%"))
|
||||
search_filters.append(
|
||||
cast(column, Text).ilike(
|
||||
f"%{_escape_like(search)}%", escape="\\"
|
||||
)
|
||||
)
|
||||
if search_filters:
|
||||
query = query.filter(or_(*search_filters))
|
||||
if custom_filters:
|
||||
|
||||
@@ -1746,7 +1746,8 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
raise cls.get_dbapi_mapped_exception(ex) from ex
|
||||
|
||||
if schema and cls.try_remove_schema_from_table_name:
|
||||
tables = {re.sub(f"^{schema}\\.", "", table) for table in tables}
|
||||
escaped_schema = re.escape(schema)
|
||||
tables = {re.sub(f"^{escaped_schema}\\.", "", table) for table in tables}
|
||||
return tables
|
||||
|
||||
@classmethod
|
||||
@@ -1774,7 +1775,8 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
raise cls.get_dbapi_mapped_exception(ex) from ex
|
||||
|
||||
if schema and cls.try_remove_schema_from_table_name:
|
||||
views = {re.sub(f"^{schema}\\.", "", view) for view in views}
|
||||
escaped_schema = re.escape(schema)
|
||||
views = {re.sub(f"^{escaped_schema}\\.", "", view) for view in views}
|
||||
return views
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -95,10 +95,11 @@ def context_addons() -> dict[str, Any]:
|
||||
return current_app.config.get("JINJA_CONTEXT_ADDONS", {})
|
||||
|
||||
|
||||
class Filter(TypedDict):
|
||||
class Filter(TypedDict, total=False):
|
||||
op: str # pylint: disable=C0103
|
||||
col: str
|
||||
val: Union[None, Any, list[Any]]
|
||||
escaped_val: Union[None, Any, list[Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -345,17 +346,57 @@ class ExtraCache:
|
||||
|
||||
return return_val
|
||||
|
||||
def _escape_value(self, val: Any) -> Any:
|
||||
"""Return a dialect-quoted form of ``val`` suitable for direct SQL
|
||||
interpolation. When no dialect is configured the value is returned
|
||||
unchanged so callers see the raw value as before. Strings are
|
||||
passed through SQLAlchemy's ``String`` literal processor (with the
|
||||
surrounding quotes stripped, mirroring ``url_param``). Lists are
|
||||
processed element-wise; non-string members are left as-is.
|
||||
"""
|
||||
if not self.dialect:
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
return String().literal_processor(dialect=self.dialect)(value=val)[1:-1]
|
||||
if isinstance(val, list):
|
||||
return [
|
||||
String().literal_processor(dialect=self.dialect)(value=v)[1:-1]
|
||||
if isinstance(v, str)
|
||||
else v
|
||||
for v in val
|
||||
]
|
||||
return val
|
||||
|
||||
def get_filters(self, column: str, remove_filter: bool = False) -> list[Filter]:
|
||||
"""Get the filters applied to the given column. In addition
|
||||
to returning values like the filter_values function
|
||||
the get_filters function returns the operator specified in the explorer UI.
|
||||
|
||||
Each filter dict additionally carries an ``escaped_val`` key when a
|
||||
SQL dialect is available. Templates that interpolate the value into
|
||||
a SQL string (for example a ``LIKE`` clause) should reference
|
||||
``escaped_val`` so the value is rendered through the dialect's
|
||||
literal processor. ``val`` continues to expose the raw value for
|
||||
non-SQL uses such as comparison, logging, or ``where_in``.
|
||||
|
||||
This is useful if:
|
||||
- you want to handle more than the IN operator in your SQL clause
|
||||
- you want to handle generating custom SQL conditions for a filter
|
||||
- you want to have the ability for filter inside the main query for speed
|
||||
purposes
|
||||
|
||||
Always use the ``where_in`` filter for list membership rather than
|
||||
building SQL by hand. The filter renders values with dialect-safe quoting
|
||||
(via SQLAlchemy's ``literal_binds`` compilation) instead of interpolating
|
||||
them directly into the SQL string.
|
||||
|
||||
.. warning::
|
||||
|
||||
Do not manually escape filter values (for example, with
|
||||
``replace("'", "''")``). Hand-rolled escaping is error-prone and easy
|
||||
to get wrong across dialects. Rely on the ``where_in`` filter so values
|
||||
are quoted safely by the engine.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
||||
@@ -379,7 +420,7 @@ class ExtraCache:
|
||||
{%- endif -%}
|
||||
{%- if filter.get('op') == 'LIKE' -%}
|
||||
AND
|
||||
full_name LIKE '{{ filter.get('val') | replace("'", "''") }}'
|
||||
full_name LIKE '{{ filter.get('escaped_val') }}'
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
UNION ALL
|
||||
@@ -446,7 +487,10 @@ class ExtraCache:
|
||||
) and not isinstance(val, list):
|
||||
val = [val]
|
||||
|
||||
filters.append({"op": op, "col": column, "val": val})
|
||||
entry: Filter = {"op": op, "col": column, "val": val}
|
||||
if self.dialect:
|
||||
entry["escaped_val"] = self._escape_value(val)
|
||||
filters.append(entry)
|
||||
|
||||
# Drill-to-detail queries send filters in native {col, op, val} format
|
||||
# rather than adhoc_filters, so get_form_data() above finds nothing.
|
||||
@@ -481,7 +525,10 @@ class ExtraCache:
|
||||
self.removed_filters.append(column)
|
||||
if column not in self.applied_filters:
|
||||
self.applied_filters.append(column)
|
||||
filters.append({"op": op, "col": column, "val": val})
|
||||
entry: Filter = {"op": op, "col": column, "val": val}
|
||||
if self.dialect:
|
||||
entry["escaped_val"] = self._escape_value(val)
|
||||
filters.append(entry)
|
||||
return filters
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
* 4. Run npm publish with appropriate access rights
|
||||
*/
|
||||
|
||||
const { spawn, execSync } = require('child_process');
|
||||
const { spawn, execSync, execFileSync } = require('child_process');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
@@ -176,7 +176,7 @@ function checkEnvironment() {
|
||||
|
||||
// Check if Superset is installed
|
||||
try {
|
||||
execSync(`${python} -c "import superset"`, {
|
||||
execFileSync(python, ['-c', 'import superset'], {
|
||||
env: { ...process.env, PYTHONPATH: supersetRoot },
|
||||
stdio: 'ignore'
|
||||
});
|
||||
|
||||
@@ -274,6 +274,125 @@ def merge_extra_form_data_filters_into_query(
|
||||
merge_form_data_filters_into_query(query, extra_query_form_data)
|
||||
|
||||
|
||||
def _deck_gl_spatial_cols(spatial: dict[str, Any] | None) -> list[str]:
|
||||
"""Return the column names referenced by a single Deck.gl spatial control."""
|
||||
if not isinstance(spatial, dict):
|
||||
return []
|
||||
spatial_type = spatial.get("type")
|
||||
if spatial_type == "latlong":
|
||||
return [c for c in [spatial.get("lonCol"), spatial.get("latCol")] if c]
|
||||
if spatial_type == "delimited":
|
||||
return [c for c in [spatial.get("lonlatCol")] if c]
|
||||
if spatial_type == "geohash":
|
||||
return [c for c in [spatial.get("geohashCol")] if c]
|
||||
return []
|
||||
|
||||
|
||||
def _is_metric_ref(value: Any) -> bool:
|
||||
"""Return True if value is a metric reference (dict or non-numeric string).
|
||||
|
||||
Deck.gl size/metric fields hold either a dict metric definition or a
|
||||
simple saved-metric string key (e.g. "count"). Scalar numeric strings
|
||||
like "100" are fixed display settings and must not be treated as metrics.
|
||||
Note: float() accepts "inf", "-inf", and "nan", so those strings would be
|
||||
excluded here too — they are not valid metric names in practice.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return True
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
float(value)
|
||||
return False
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _deck_gl_null_filters(form_data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Build IS NOT NULL simple filters for Deck.gl spatial and data columns.
|
||||
|
||||
Mirrors BaseDeckGLViz.add_null_filters() behavior: spatial control columns,
|
||||
line_column, and the geojson column are filtered for non-null values by
|
||||
default.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for key in ("spatial", "start_spatial", "end_spatial"):
|
||||
for col in _deck_gl_spatial_cols(form_data.get(key)):
|
||||
if col not in seen:
|
||||
seen.add(col)
|
||||
result.append({"col": col, "op": "IS NOT NULL", "val": ""})
|
||||
for field in ("line_column", "geojson"):
|
||||
data_col = form_data.get(field)
|
||||
if isinstance(data_col, str) and data_col and data_col not in seen:
|
||||
seen.add(data_col)
|
||||
result.append({"col": data_col, "op": "IS NOT NULL", "val": ""})
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_deck_gl_metrics(
|
||||
form_data: dict[str, Any], viz_type: str = ""
|
||||
) -> list[Any]:
|
||||
"""Extract metrics for Deck.gl chart types.
|
||||
|
||||
deck_geojson.query_obj() forces metrics=[] regardless of form_data.
|
||||
For other types, size/metric values are included when they are metric
|
||||
references (dicts or non-numeric strings); numeric scalars like "100"
|
||||
are fixed display settings and are excluded.
|
||||
deck_scatter and deck_polygon can additionally store metric-backed
|
||||
values in point_radius_fixed (radius for scatter, elevation for polygon).
|
||||
"""
|
||||
if viz_type == "deck_geojson":
|
||||
return []
|
||||
metrics: list[Any] = []
|
||||
for field in ("size", "metric"):
|
||||
m = form_data.get(field)
|
||||
if _is_metric_ref(m):
|
||||
metrics.append(m)
|
||||
prf = form_data.get("point_radius_fixed")
|
||||
if isinstance(prf, dict) and prf.get("type") == "metric":
|
||||
value = prf.get("value")
|
||||
if value:
|
||||
metrics.append(value)
|
||||
elif isinstance(prf, str) and _is_metric_ref(prf):
|
||||
# Legacy deck_scatter: point_radius_fixed as a bare non-numeric metric key
|
||||
logger.debug("Legacy point_radius_fixed string metric encountered: %s", prf)
|
||||
metrics.append(prf)
|
||||
return metrics
|
||||
|
||||
|
||||
def resolve_deck_gl_columns(form_data: dict[str, Any]) -> list[str]:
|
||||
"""Extract SQL column names for Deck.gl chart types from form_data.
|
||||
|
||||
Deck.gl charts use spatial controls (lat/lon pairs, geohash, etc.)
|
||||
rather than the standard metrics/groupby structure. This function
|
||||
maps those spatial control configs to the actual column names
|
||||
needed by the SQL query.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
columns: list[str] = []
|
||||
|
||||
def _add(col: str | None) -> None:
|
||||
if col and isinstance(col, str) and col not in seen:
|
||||
seen.add(col)
|
||||
columns.append(col)
|
||||
|
||||
# Most Deck.gl types use "spatial"; arc charts use start/end spatial
|
||||
for key in ("spatial", "start_spatial", "end_spatial"):
|
||||
for col in _deck_gl_spatial_cols(form_data.get(key)):
|
||||
_add(col)
|
||||
|
||||
# deck_path / deck_polygon use a line column; deck_geojson uses geojson
|
||||
for field in ("line_column", "geojson", "dimension"):
|
||||
_add(form_data.get(field))
|
||||
|
||||
for col in form_data.get("js_columns") or []:
|
||||
if isinstance(col, str):
|
||||
_add(col)
|
||||
|
||||
return columns
|
||||
|
||||
|
||||
def resolve_metrics(form_data: dict[str, Any], viz_type: str) -> list[Any]:
|
||||
"""Extract metrics from form_data, handling chart-type-specific fields."""
|
||||
if viz_type == "bubble":
|
||||
@@ -412,6 +531,12 @@ def _build_mixed_timeseries_secondary(
|
||||
return qd
|
||||
|
||||
|
||||
# Deck.gl viz types that conditionally set is_timeseries from time_grain_sqla
|
||||
_DECK_TIMESERIES_VIZ_TYPES: frozenset[str] = frozenset(
|
||||
{"deck_arc", "deck_path", "deck_polygon", "deck_scatter", "deck_screengrid"}
|
||||
)
|
||||
|
||||
|
||||
def build_query_dicts_from_form_data(
|
||||
form_data: dict[str, Any],
|
||||
datasource_id: Any,
|
||||
@@ -437,6 +562,34 @@ def build_query_dicts_from_form_data(
|
||||
or (getattr(chart, "viz_type", "") if chart else "")
|
||||
or ""
|
||||
)
|
||||
|
||||
# Deck.gl charts use spatial column configs rather than the standard
|
||||
# metrics / groupby fields. Extract columns from the spatial controls.
|
||||
if viz_type.startswith("deck_"):
|
||||
deck_columns = resolve_deck_gl_columns(form_data)
|
||||
deck_metrics = _resolve_deck_gl_metrics(form_data, viz_type)
|
||||
qd = _build_single_query_dict(
|
||||
form_data,
|
||||
deck_columns,
|
||||
deck_metrics,
|
||||
row_limit=row_limit,
|
||||
order_desc=order_desc,
|
||||
)
|
||||
if deck_metrics:
|
||||
# Mirror BaseDeckGLViz.query_obj(): order by first metric descending
|
||||
qd["orderby"] = [(deck_metrics[0], not form_data.get("order_desc", True))]
|
||||
if viz_type in _DECK_TIMESERIES_VIZ_TYPES and (
|
||||
time_grain := form_data.get("time_grain_sqla")
|
||||
):
|
||||
qd["is_timeseries"] = True
|
||||
qd["granularity"] = form_data.get("granularity_sqla")
|
||||
qd.setdefault("extras", {})["time_grain_sqla"] = time_grain
|
||||
if form_data.get("filter_nulls", True):
|
||||
null_filters = _deck_gl_null_filters(form_data)
|
||||
if null_filters:
|
||||
qd["filters"] = [*(qd.get("filters") or []), *null_filters]
|
||||
return [qd]
|
||||
|
||||
is_timeseries = (
|
||||
viz_type.startswith("echarts_timeseries") or viz_type == "mixed_timeseries"
|
||||
)
|
||||
|
||||
@@ -522,31 +522,10 @@ async def get_chart_data( # noqa: C901
|
||||
# groupby-like fields (entity, series, columns):
|
||||
# world_map, treemap_v2, sunburst_v2, gauge_chart
|
||||
# Bubble charts use x/y/size as separate metric fields.
|
||||
# Deck.gl charts (deck_arc, deck_scatter, etc.) use spatial
|
||||
# column configs (lat/lon, geohash, etc.) instead.
|
||||
viz_type = chart.viz_type or ""
|
||||
|
||||
# Deck.gl chart types store spatial data (lat/lon)
|
||||
# rather than traditional metrics/groupby. They
|
||||
# require a saved query_context to retrieve data.
|
||||
# Match by prefix to cover all current and future
|
||||
# deck.gl viz types (deck_arc, deck_scatter, etc.).
|
||||
if viz_type.startswith("deck_"):
|
||||
await ctx.warning(
|
||||
"Chart %s is a deck.gl visualization (%s) with no "
|
||||
"saved query_context. Data retrieval requires "
|
||||
"re-saving the chart in Superset." % (chart.id, viz_type)
|
||||
)
|
||||
return ChartError(
|
||||
error=(
|
||||
f"Chart {chart.id} is a deck.gl visualization "
|
||||
f"(type: {viz_type}) with no saved query_context. "
|
||||
f"Deck.gl charts use spatial data (lat/lon) that "
|
||||
f"cannot be reconstructed from form_data alone. "
|
||||
f"Please open this chart in Superset and re-save "
|
||||
f"it to generate a query_context."
|
||||
),
|
||||
error_type="MissingQueryContext",
|
||||
)
|
||||
|
||||
fallback_queries = build_query_dicts_from_form_data(
|
||||
form_data,
|
||||
chart.datasource_id,
|
||||
|
||||
@@ -1449,6 +1449,36 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
|
||||
if is_alias_used_in_orderby(col):
|
||||
col.name = f"{col.name}__"
|
||||
|
||||
def _raise_for_disallowed_sql(self, sql: str) -> None:
|
||||
"""
|
||||
Mirror the DISALLOWED_SQL_* gate that sql_lab.execute_sql_statement
|
||||
enforces so both query surfaces honour the same denylist.
|
||||
"""
|
||||
engine = self.db_engine_spec.engine
|
||||
disallowed_functions = app.config["DISALLOWED_SQL_FUNCTIONS"].get(engine, set())
|
||||
disallowed_tables = app.config["DISALLOWED_SQL_TABLES"].get(engine, set())
|
||||
if not (disallowed_functions or disallowed_tables):
|
||||
return
|
||||
|
||||
parsed_script = SQLScript(sql, engine=engine)
|
||||
if disallowed_functions and parsed_script.check_functions_present(
|
||||
disallowed_functions
|
||||
):
|
||||
raise SupersetDisallowedSQLFunctionException(disallowed_functions)
|
||||
if disallowed_tables and parsed_script.check_tables_present(disallowed_tables):
|
||||
# Report only the tables actually found in the query, mirroring the
|
||||
# canonical execution-time gate so the user-facing error doesn't
|
||||
# echo the operator's full denylist.
|
||||
present_tables = {
|
||||
table.table.lower()
|
||||
for statement in parsed_script.statements
|
||||
for table in statement.tables
|
||||
}
|
||||
found_tables = {
|
||||
table for table in disallowed_tables if table.lower() in present_tables
|
||||
}
|
||||
raise SupersetDisallowedSQLTableException(found_tables or disallowed_tables)
|
||||
|
||||
def query(self, query_obj: QueryObjectDict) -> QueryResult:
|
||||
"""
|
||||
Executes the query and returns a dataframe.
|
||||
@@ -1459,6 +1489,9 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
|
||||
qry_start_dttm = datetime.now()
|
||||
query_str_ext = self.get_query_str_extended(query_obj)
|
||||
sql = query_str_ext.sql
|
||||
|
||||
self._raise_for_disallowed_sql(sql)
|
||||
|
||||
status = QueryStatus.SUCCESS
|
||||
errors = None
|
||||
error_message = None
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
Babel==2.9.1
|
||||
# Keep Babel in sync with requirements/base.txt — CI extracts/updates the
|
||||
# catalogs with that version (via requirements/development.txt), so a different
|
||||
# pin here makes local `babel_update.sh` runs produce spurious reformatting diffs.
|
||||
Babel==2.17.0
|
||||
jinja2==3.1.6
|
||||
polib>=1.2.0
|
||||
|
||||
@@ -809,6 +809,75 @@ def test_base_dao_list_search(user_with_data: Session) -> None:
|
||||
user_with_data.commit()
|
||||
|
||||
|
||||
def test_base_dao_list_search_wildcard_injection(user_with_data: Session) -> None:
|
||||
"""Verify that search='%' is treated literally and does not match all rows."""
|
||||
users = []
|
||||
for i in range(3):
|
||||
user = User(
|
||||
id=420 + i,
|
||||
username=f"nowildcard_{i}",
|
||||
first_name=f"NoWild{i}",
|
||||
last_name="Card",
|
||||
email=f"nowild{i}@example.com",
|
||||
active=True,
|
||||
)
|
||||
users.append(user)
|
||||
user_with_data.add(user)
|
||||
user_with_data.commit()
|
||||
|
||||
# '%' must be escaped; it should not match any of our plain-name users
|
||||
results, total = UserDAO.list(search="%", search_columns=["username", "first_name"])
|
||||
result_usernames = {r.username for r in results}
|
||||
for user in users:
|
||||
assert user.username not in result_usernames, (
|
||||
f"search='%' should not match '{user.username}' — LIKE wildcard injection"
|
||||
)
|
||||
|
||||
for user in users:
|
||||
user_with_data.delete(user)
|
||||
user_with_data.commit()
|
||||
|
||||
|
||||
def test_base_dao_list_column_operator_wildcard_injection(
|
||||
user_with_data: Session,
|
||||
) -> None:
|
||||
"""Verify that column_operators with '%' values are treated literally."""
|
||||
users = []
|
||||
for i in range(3):
|
||||
user = User(
|
||||
id=430 + i,
|
||||
username=f"opwildcard_{i}",
|
||||
first_name=f"OpWild{i}",
|
||||
last_name="Card",
|
||||
email=f"opwild{i}@example.com",
|
||||
active=True,
|
||||
)
|
||||
users.append(user)
|
||||
user_with_data.add(user)
|
||||
user_with_data.commit()
|
||||
|
||||
# Each wildcard operator with '%' as the value must not match plain-name rows
|
||||
for opr in (
|
||||
ColumnOperatorEnum.ct,
|
||||
ColumnOperatorEnum.like,
|
||||
ColumnOperatorEnum.ilike,
|
||||
ColumnOperatorEnum.sw,
|
||||
ColumnOperatorEnum.ew,
|
||||
):
|
||||
results, _ = UserDAO.list(
|
||||
column_operators=[ColumnOperator(col="username", opr=opr, value="%")]
|
||||
)
|
||||
result_usernames = {r.username for r in results}
|
||||
for user in users:
|
||||
assert user.username not in result_usernames, (
|
||||
f"opr={opr!r} with value='%' should not match '{user.username}'"
|
||||
)
|
||||
|
||||
for user in users:
|
||||
user_with_data.delete(user)
|
||||
user_with_data.commit()
|
||||
|
||||
|
||||
def test_base_dao_list_custom_filter(user_with_data: Session) -> None:
|
||||
"""Test BaseDAO.list with custom filters."""
|
||||
# Create users with specific attributes
|
||||
|
||||
@@ -31,6 +31,8 @@ from superset.daos.dataset import DatasetDAO
|
||||
from superset.daos.exceptions import DatasourceNotFound
|
||||
from superset.exceptions import (
|
||||
OAuth2RedirectError,
|
||||
SupersetDisallowedSQLFunctionException,
|
||||
SupersetDisallowedSQLTableException,
|
||||
SupersetSecurityException,
|
||||
)
|
||||
from superset.models.core import Database
|
||||
@@ -81,6 +83,116 @@ def test_query_bubbles_errors(mocker: MockerFixture) -> None:
|
||||
sqla_table.query(query_obj)
|
||||
|
||||
|
||||
def _query_obj() -> QueryObjectDict:
|
||||
return {
|
||||
"granularity": None,
|
||||
"from_dttm": None,
|
||||
"to_dttm": None,
|
||||
"groupby": ["id"],
|
||||
"metrics": [],
|
||||
"is_timeseries": False,
|
||||
"filter": [],
|
||||
}
|
||||
|
||||
|
||||
def _build_sqla_table_for_query(
|
||||
mocker: MockerFixture, sql: str, engine: str = "postgresql"
|
||||
) -> SqlaTable:
|
||||
db_engine_spec = mocker.MagicMock()
|
||||
db_engine_spec.engine = engine
|
||||
database = mocker.MagicMock()
|
||||
database.db_engine_spec = db_engine_spec
|
||||
sqla_table = SqlaTable(
|
||||
table_name="my_sqla_table",
|
||||
columns=[],
|
||||
metrics=[],
|
||||
database=database,
|
||||
)
|
||||
mocker.patch.object(
|
||||
SqlaTable,
|
||||
"db_engine_spec",
|
||||
new=property(lambda self: db_engine_spec),
|
||||
)
|
||||
mocker.patch.object(
|
||||
sqla_table,
|
||||
"get_query_str_extended",
|
||||
return_value=mocker.MagicMock(sql=sql, labels_expected=[]),
|
||||
)
|
||||
return sqla_table
|
||||
|
||||
|
||||
def test_query_blocks_disallowed_function_on_chart_data_path(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
mocker.patch.dict(
|
||||
"flask.current_app.config",
|
||||
{
|
||||
"DISALLOWED_SQL_FUNCTIONS": {"postgresql": {"version"}},
|
||||
"DISALLOWED_SQL_TABLES": {},
|
||||
},
|
||||
clear=False,
|
||||
)
|
||||
sqla_table = _build_sqla_table_for_query(mocker, "SELECT version()")
|
||||
with pytest.raises(SupersetDisallowedSQLFunctionException):
|
||||
sqla_table.query(_query_obj())
|
||||
sqla_table.database.get_df.assert_not_called() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_query_blocks_disallowed_table_on_chart_data_path(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
mocker.patch.dict(
|
||||
"flask.current_app.config",
|
||||
{
|
||||
"DISALLOWED_SQL_FUNCTIONS": {},
|
||||
"DISALLOWED_SQL_TABLES": {"postgresql": {"pg_authid"}},
|
||||
},
|
||||
clear=False,
|
||||
)
|
||||
sqla_table = _build_sqla_table_for_query(mocker, "SELECT rolname FROM pg_authid")
|
||||
with pytest.raises(SupersetDisallowedSQLTableException):
|
||||
sqla_table.query(_query_obj())
|
||||
sqla_table.database.get_df.assert_not_called() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_query_disallowed_table_error_reports_only_matched_tables(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
mocker.patch.dict(
|
||||
"flask.current_app.config",
|
||||
{
|
||||
"DISALLOWED_SQL_FUNCTIONS": {},
|
||||
"DISALLOWED_SQL_TABLES": {
|
||||
"postgresql": {"pg_authid", "pg_shadow", "pg_stat_activity"}
|
||||
},
|
||||
},
|
||||
clear=False,
|
||||
)
|
||||
sqla_table = _build_sqla_table_for_query(mocker, "SELECT rolname FROM pg_authid")
|
||||
with pytest.raises(SupersetDisallowedSQLTableException) as excinfo:
|
||||
sqla_table.query(_query_obj())
|
||||
message = str(excinfo.value)
|
||||
assert "pg_authid" in message
|
||||
assert "pg_shadow" not in message
|
||||
assert "pg_stat_activity" not in message
|
||||
|
||||
|
||||
def test_query_allows_benign_sql_on_chart_data_path(mocker: MockerFixture) -> None:
|
||||
mocker.patch.dict(
|
||||
"flask.current_app.config",
|
||||
{
|
||||
"DISALLOWED_SQL_FUNCTIONS": {"postgresql": {"version"}},
|
||||
"DISALLOWED_SQL_TABLES": {"postgresql": {"pg_authid"}},
|
||||
},
|
||||
clear=False,
|
||||
)
|
||||
sqla_table = _build_sqla_table_for_query(mocker, "SELECT id FROM my_sqla_table")
|
||||
sqla_table.database.get_df.return_value = pd.DataFrame() # type: ignore[attr-defined]
|
||||
result = sqla_table.query(_query_obj())
|
||||
sqla_table.database.get_df.assert_called_once() # type: ignore[attr-defined]
|
||||
assert result is not None
|
||||
|
||||
|
||||
def test_permissions_without_catalog() -> None:
|
||||
"""
|
||||
Test permissions when the table has no catalog.
|
||||
@@ -276,7 +388,7 @@ def test_dataset_uniqueness(session: Session) -> None:
|
||||
|
||||
def test_normalize_prequery_result_type_custom_sql() -> None:
|
||||
"""
|
||||
Test that the `_normalize_prequery_result_type` can hanndle custom SQL.
|
||||
Test that the `_normalize_prequery_result_type` can handle custom SQL.
|
||||
"""
|
||||
sqla_table = SqlaTable(
|
||||
table_name="my_sqla_table",
|
||||
|
||||
@@ -1283,3 +1283,59 @@ def test_start_oauth2_dance_falls_back_to_url_for(mocker: MockerFixture) -> None
|
||||
error = exc_info.value.error
|
||||
|
||||
assert error.extra["redirect_uri"] == fallback_uri
|
||||
|
||||
|
||||
def test_get_table_names_strips_schema_with_regex_metacharacters(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
Test that get_table_names strips a schema prefix containing regex
|
||||
metacharacters without raising and without mangling unrelated names.
|
||||
"""
|
||||
schema = "a.b(c)"
|
||||
|
||||
inspector = mocker.MagicMock()
|
||||
inspector.get_table_names.return_value = [
|
||||
f"{schema}.orders",
|
||||
"axbc.other",
|
||||
]
|
||||
|
||||
database = mocker.MagicMock()
|
||||
|
||||
spec = BaseEngineSpec
|
||||
mocker.patch.object(spec, "try_remove_schema_from_table_name", True)
|
||||
|
||||
tables = spec.get_table_names(database, inspector, schema)
|
||||
|
||||
# The real schema prefix is stripped; the look-alike name is left intact
|
||||
# because the metacharacters are escaped before being used as a regex.
|
||||
# "axbc.other" would match the old unescaped pattern ^a.b(c)\. and be
|
||||
# incorrectly stripped — the escaped version correctly preserves it.
|
||||
assert tables == {"orders", "axbc.other"}
|
||||
|
||||
|
||||
def test_get_view_names_strips_schema_with_regex_metacharacters(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
Test that get_view_names strips a schema prefix containing regex
|
||||
metacharacters without raising and without mangling unrelated names.
|
||||
"""
|
||||
schema = "a.b(c)"
|
||||
|
||||
inspector = mocker.MagicMock()
|
||||
inspector.get_view_names.return_value = [
|
||||
f"{schema}.report",
|
||||
"axbc.other",
|
||||
]
|
||||
|
||||
database = mocker.MagicMock()
|
||||
|
||||
spec = BaseEngineSpec
|
||||
mocker.patch.object(spec, "try_remove_schema_from_table_name", True)
|
||||
|
||||
views = spec.get_view_names(database, inspector, schema)
|
||||
|
||||
# "axbc.other" would match the old unescaped pattern ^a.b(c)\. and be
|
||||
# incorrectly stripped — the escaped version correctly preserves it.
|
||||
assert views == {"report", "axbc.other"}
|
||||
|
||||
@@ -354,6 +354,124 @@ def test_filter_values_query_context_filters() -> None:
|
||||
assert cache.applied_filters == ["name"]
|
||||
|
||||
|
||||
def test_get_filters_escaped_val_string_adhoc() -> None:
|
||||
"""
|
||||
``get_filters`` exposes an ``escaped_val`` companion for string values
|
||||
when a dialect is configured, while leaving ``val`` raw.
|
||||
"""
|
||||
with current_app.test_request_context(
|
||||
data={
|
||||
"form_data": json.dumps(
|
||||
{
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "O'Brien",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "LIKE",
|
||||
"subject": "name",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
}
|
||||
):
|
||||
cache = ExtraCache(dialect=dialect())
|
||||
result = cache.get_filters("name")
|
||||
assert result == [
|
||||
{
|
||||
"op": "LIKE",
|
||||
"col": "name",
|
||||
"val": "O'Brien",
|
||||
"escaped_val": "O''Brien",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_filters_escaped_val_list_adhoc() -> None:
|
||||
"""
|
||||
``get_filters`` produces an ``escaped_val`` list with every string
|
||||
member dialect-escaped; non-string members pass through untouched.
|
||||
"""
|
||||
with current_app.test_request_context(
|
||||
data={
|
||||
"form_data": json.dumps(
|
||||
{
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": ["O'Brien", "Smith"],
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "in",
|
||||
"subject": "name",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
}
|
||||
):
|
||||
cache = ExtraCache(dialect=dialect())
|
||||
result = cache.get_filters("name")
|
||||
assert result == [
|
||||
{
|
||||
"op": "IN",
|
||||
"col": "name",
|
||||
"val": ["O'Brien", "Smith"],
|
||||
"escaped_val": ["O''Brien", "Smith"],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_filters_escaped_val_query_context_filters() -> None:
|
||||
"""
|
||||
The ``escaped_val`` companion is also populated when filters arrive via
|
||||
the drill-to-detail ``query_context_filters`` path.
|
||||
"""
|
||||
cache = ExtraCache(
|
||||
dialect=dialect(),
|
||||
query_context_filters=[
|
||||
{"col": "name", "op": "LIKE", "val": "O'Brien"},
|
||||
],
|
||||
)
|
||||
assert cache.get_filters("name") == [
|
||||
{
|
||||
"op": "LIKE",
|
||||
"col": "name",
|
||||
"val": "O'Brien",
|
||||
"escaped_val": "O''Brien",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_filters_no_escaped_val_without_dialect() -> None:
|
||||
"""
|
||||
Without a dialect ``get_filters`` returns the original schema, with no
|
||||
``escaped_val`` key — preserving backwards compatibility for callers
|
||||
that did not opt into a dialect-aware processor.
|
||||
"""
|
||||
with current_app.test_request_context(
|
||||
data={
|
||||
"form_data": json.dumps(
|
||||
{
|
||||
"adhoc_filters": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"comparator": "O'Brien",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "LIKE",
|
||||
"subject": "name",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
}
|
||||
):
|
||||
cache = ExtraCache()
|
||||
result = cache.get_filters("name")
|
||||
assert result == [{"op": "LIKE", "col": "name", "val": "O'Brien"}]
|
||||
assert "escaped_val" not in result[0]
|
||||
|
||||
|
||||
def test_url_param_query() -> None:
|
||||
"""
|
||||
Test the ``url_param`` macro.
|
||||
|
||||
@@ -18,6 +18,9 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from superset.mcp_service.chart.chart_helpers import (
|
||||
_deck_gl_null_filters,
|
||||
_is_metric_ref,
|
||||
_resolve_deck_gl_metrics,
|
||||
apply_form_data_filters_to_query,
|
||||
build_query_dicts_from_form_data,
|
||||
extract_form_data_key_from_url,
|
||||
@@ -26,6 +29,7 @@ from superset.mcp_service.chart.chart_helpers import (
|
||||
merge_extra_form_data_filters_into_query,
|
||||
merge_form_data_filters_into_query,
|
||||
prepare_form_data_for_query,
|
||||
resolve_deck_gl_columns,
|
||||
)
|
||||
|
||||
|
||||
@@ -285,3 +289,650 @@ def test_merge_extra_form_data_filters_into_query_adds_only_extra_predicates(
|
||||
assert query["time_range"] == "No filter"
|
||||
assert query["granularity"] == "updated_at"
|
||||
assert query["time_grain_sqla"] == "P1D"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_deck_gl_columns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_latlong():
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "longitude", "latCol": "latitude"},
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["longitude", "latitude"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_delimited():
|
||||
form_data = {
|
||||
"spatial": {"type": "delimited", "lonlatCol": "coords"},
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["coords"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_geohash():
|
||||
form_data = {
|
||||
"spatial": {"type": "geohash", "geohashCol": "geo"},
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["geo"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_arc_start_end():
|
||||
form_data = {
|
||||
"start_spatial": {
|
||||
"type": "latlong",
|
||||
"lonCol": "start_lon",
|
||||
"latCol": "start_lat",
|
||||
},
|
||||
"end_spatial": {"type": "latlong", "lonCol": "end_lon", "latCol": "end_lat"},
|
||||
}
|
||||
cols = resolve_deck_gl_columns(form_data)
|
||||
assert cols == ["start_lon", "start_lat", "end_lon", "end_lat"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_path_line_column():
|
||||
form_data = {
|
||||
"line_column": "path_wkt",
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["path_wkt"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_geojson():
|
||||
form_data = {
|
||||
"geojson": "geom_col",
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["geom_col"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_with_dimension_and_js_columns():
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"dimension": "category",
|
||||
"js_columns": ["name", "value"],
|
||||
}
|
||||
cols = resolve_deck_gl_columns(form_data)
|
||||
assert "lon" in cols
|
||||
assert "lat" in cols
|
||||
assert "category" in cols
|
||||
assert "name" in cols
|
||||
assert "value" in cols
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_deduplicates():
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"dimension": "lon", # same as lonCol — should not duplicate
|
||||
}
|
||||
cols = resolve_deck_gl_columns(form_data)
|
||||
assert cols.count("lon") == 1
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_empty():
|
||||
assert resolve_deck_gl_columns({}) == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_ignores_non_string_js_columns():
|
||||
form_data = {
|
||||
"js_columns": [42, None, "valid_col"],
|
||||
}
|
||||
assert resolve_deck_gl_columns(form_data) == ["valid_col"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_query_dicts_from_form_data — Deck.gl branch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_latlong(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["columns"] == ["lon", "lat"]
|
||||
assert queries[0]["metrics"] == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_with_size_metric(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
metric = {
|
||||
"expressionType": "SIMPLE",
|
||||
"column": {"column_name": "sales"},
|
||||
"aggregate": "SUM",
|
||||
}
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"size": metric,
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["columns"] == ["lon", "lat"]
|
||||
assert queries[0]["metrics"] == [metric]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_arc(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_arc",
|
||||
"start_spatial": {
|
||||
"type": "latlong",
|
||||
"lonCol": "origin_lon",
|
||||
"latCol": "origin_lat",
|
||||
},
|
||||
"end_spatial": {"type": "latlong", "lonCol": "dest_lon", "latCol": "dest_lat"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["columns"] == ["origin_lon", "origin_lat", "dest_lon", "dest_lat"]
|
||||
assert queries[0]["metrics"] == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_geojson(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_geojson",
|
||||
"geojson": "geometry",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["columns"] == ["geometry"]
|
||||
assert queries[0]["metrics"] == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_hex_geohash(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_hex",
|
||||
"spatial": {"type": "geohash", "geohashCol": "geohash"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["columns"] == ["geohash"]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_path_with_row_limit(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_path",
|
||||
"line_column": "path_col",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table", row_limit=50)
|
||||
|
||||
assert queries[0]["columns"] == ["path_col"]
|
||||
assert queries[0]["row_limit"] == 50
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_deck_gl_columns — display-only fields excluded
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_ignores_tooltip_contents():
|
||||
# tooltip_contents are display-only; BaseDeckGLViz.query_obj() does not
|
||||
# include them in columns/groupby, so the fallback should not either.
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"tooltip_contents": ["name", "category"],
|
||||
}
|
||||
cols = resolve_deck_gl_columns(form_data)
|
||||
assert "name" not in cols
|
||||
assert "category" not in cols
|
||||
|
||||
|
||||
def test_resolve_deck_gl_columns_ignores_cross_filter_column():
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"cross_filter_column": "region",
|
||||
}
|
||||
cols = resolve_deck_gl_columns(form_data)
|
||||
assert "region" not in cols
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _is_metric_ref
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_metric_ref_dict():
|
||||
assert _is_metric_ref({"expressionType": "SIMPLE"}) is True
|
||||
|
||||
|
||||
def test_is_metric_ref_string_key():
|
||||
assert _is_metric_ref("count") is True
|
||||
assert _is_metric_ref("sum__sales") is True
|
||||
|
||||
|
||||
def test_is_metric_ref_numeric_string_excluded():
|
||||
assert _is_metric_ref("100") is False
|
||||
assert _is_metric_ref("3.14") is False
|
||||
assert _is_metric_ref("0") is False
|
||||
|
||||
|
||||
def test_is_metric_ref_integer_excluded():
|
||||
assert _is_metric_ref(100) is False
|
||||
|
||||
|
||||
def test_is_metric_ref_none_and_empty():
|
||||
assert _is_metric_ref(None) is False
|
||||
assert _is_metric_ref("") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _resolve_deck_gl_metrics (Fix 2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_no_metrics():
|
||||
assert _resolve_deck_gl_metrics({}) == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_size_field():
|
||||
metric = {"expressionType": "SIMPLE", "aggregate": "COUNT", "column": None}
|
||||
result = _resolve_deck_gl_metrics({"size": metric})
|
||||
assert result == [metric]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_metric_field():
|
||||
metric = {"expressionType": "SIMPLE", "aggregate": "SUM"}
|
||||
result = _resolve_deck_gl_metrics({"metric": metric})
|
||||
assert result == [metric]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_point_radius_fixed_metric():
|
||||
prf_metric = {"expressionType": "SIMPLE", "aggregate": "AVG"}
|
||||
prf = {"type": "metric", "value": prf_metric}
|
||||
result = _resolve_deck_gl_metrics({"point_radius_fixed": prf})
|
||||
assert result == [prf_metric]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_point_radius_fixed_not_metric():
|
||||
prf = {"type": "fix", "value": 100}
|
||||
result = _resolve_deck_gl_metrics({"point_radius_fixed": prf})
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_polygon_both_metric_and_prf():
|
||||
base_metric = {"expressionType": "SIMPLE", "aggregate": "SUM"}
|
||||
elevation_metric = {"expressionType": "SIMPLE", "aggregate": "AVG"}
|
||||
prf = {"type": "metric", "value": elevation_metric}
|
||||
result = _resolve_deck_gl_metrics(
|
||||
{"metric": base_metric, "point_radius_fixed": prf}
|
||||
)
|
||||
assert result == [base_metric, elevation_metric]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_geojson_returns_empty():
|
||||
# deck_geojson.query_obj() forces metrics=[] regardless of form_data
|
||||
metric = {"expressionType": "SIMPLE", "aggregate": "SUM"}
|
||||
result = _resolve_deck_gl_metrics(
|
||||
{"size": metric, "metric": metric}, "deck_geojson"
|
||||
)
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_scalar_size_excluded():
|
||||
# Numeric string size values (fixed display settings) must not be metrics
|
||||
result = _resolve_deck_gl_metrics({"size": "100"}, "deck_hex")
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_integer_size_excluded():
|
||||
result = _resolve_deck_gl_metrics({"size": 100}, "deck_path")
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_string_metric_included():
|
||||
# Non-numeric string metrics (saved metric keys) must be preserved
|
||||
result = _resolve_deck_gl_metrics({"size": "count"}, "deck_hex")
|
||||
assert result == ["count"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_string_metric_field():
|
||||
result = _resolve_deck_gl_metrics({"metric": "sum__sales"}, "deck_arc")
|
||||
assert result == ["sum__sales"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_string_point_radius_fixed():
|
||||
# Legacy deck_scatter: point_radius_fixed as a bare metric key string
|
||||
result = _resolve_deck_gl_metrics({"point_radius_fixed": "count"}, "deck_scatter")
|
||||
assert result == ["count"]
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_numeric_point_radius_fixed_excluded():
|
||||
# Numeric string point_radius_fixed is a fixed pixel radius, not a metric
|
||||
result = _resolve_deck_gl_metrics({"point_radius_fixed": "100"}, "deck_scatter")
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_resolve_deck_gl_metrics_non_string_point_radius_fixed_excluded():
|
||||
# Non-string point_radius_fixed values (int, None, list) are excluded by
|
||||
# the isinstance(prf, str) guard in the elif branch
|
||||
assert _resolve_deck_gl_metrics({"point_radius_fixed": 100}, "deck_scatter") == []
|
||||
assert _resolve_deck_gl_metrics({"point_radius_fixed": None}, "deck_scatter") == []
|
||||
assert (
|
||||
_resolve_deck_gl_metrics({"point_radius_fixed": ["bad"]}, "deck_scatter") == []
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _deck_gl_null_filters (Fix 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_deck_gl_null_filters_latlong():
|
||||
form_data = {
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
}
|
||||
result = _deck_gl_null_filters(form_data)
|
||||
assert result == [
|
||||
{"col": "lon", "op": "IS NOT NULL", "val": ""},
|
||||
{"col": "lat", "op": "IS NOT NULL", "val": ""},
|
||||
]
|
||||
|
||||
|
||||
def test_deck_gl_null_filters_arc_start_end():
|
||||
form_data = {
|
||||
"start_spatial": {"type": "latlong", "lonCol": "s_lon", "latCol": "s_lat"},
|
||||
"end_spatial": {"type": "latlong", "lonCol": "e_lon", "latCol": "e_lat"},
|
||||
}
|
||||
result = _deck_gl_null_filters(form_data)
|
||||
assert result == [
|
||||
{"col": "s_lon", "op": "IS NOT NULL", "val": ""},
|
||||
{"col": "s_lat", "op": "IS NOT NULL", "val": ""},
|
||||
{"col": "e_lon", "op": "IS NOT NULL", "val": ""},
|
||||
{"col": "e_lat", "op": "IS NOT NULL", "val": ""},
|
||||
]
|
||||
|
||||
|
||||
def test_deck_gl_null_filters_line_column():
|
||||
form_data = {"line_column": "path_col"}
|
||||
result = _deck_gl_null_filters(form_data)
|
||||
assert result == [{"col": "path_col", "op": "IS NOT NULL", "val": ""}]
|
||||
|
||||
|
||||
def test_deck_gl_null_filters_empty():
|
||||
assert _deck_gl_null_filters({}) == []
|
||||
|
||||
|
||||
def test_deck_gl_null_filters_geojson_column():
|
||||
# geojson column gets an IS NOT NULL filter just like spatial columns
|
||||
form_data = {"geojson": "geometry"}
|
||||
assert _deck_gl_null_filters(form_data) == [
|
||||
{"col": "geometry", "op": "IS NOT NULL", "val": ""}
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_query_dicts_from_form_data — null filters behavior (Fix 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_adds_null_filters_by_default(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert {"col": "lon", "op": "IS NOT NULL", "val": ""} in queries[0]["filters"]
|
||||
assert {"col": "lat", "op": "IS NOT NULL", "val": ""} in queries[0]["filters"]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_filter_nulls_false(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"filter_nulls": False,
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
null_filters = [
|
||||
f for f in queries[0].get("filters", []) if f.get("op") == "IS NOT NULL"
|
||||
]
|
||||
assert null_filters == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_point_radius_fixed_metric(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
radius_metric = {
|
||||
"expressionType": "SIMPLE",
|
||||
"aggregate": "AVG",
|
||||
"column": {"column_name": "radius"},
|
||||
}
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"point_radius_fixed": {"type": "metric", "value": radius_metric},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["metrics"] == [radius_metric]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_geojson_scalar_size_produces_no_metrics(monkeypatch):
|
||||
# Regression: deck_geojson fixture has size='100' (scalar, not a metric).
|
||||
# The fallback must produce metrics=[] to match DeckGeoJson.query_obj().
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_geojson",
|
||||
"geojson": "geometry",
|
||||
"size": "100",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["metrics"] == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_path_scalar_size_produces_no_metrics(monkeypatch):
|
||||
# deck_path fixture also has size='100' — scalar must not become a metric.
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_path",
|
||||
"line_column": "path_col",
|
||||
"size": "100",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["metrics"] == []
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_geojson_adds_geojson_null_filter(monkeypatch):
|
||||
# deck_geojson should add IS NOT NULL on the geojson column when filter_nulls
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_geojson",
|
||||
"geojson": "geometry_col",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert {"col": "geometry_col", "op": "IS NOT NULL", "val": ""} in queries[0][
|
||||
"filters"
|
||||
]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_hex_string_metric(monkeypatch):
|
||||
# Non-numeric string size (saved metric key) must be included as a metric
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_hex",
|
||||
"spatial": {"type": "geohash", "geohashCol": "geo"},
|
||||
"size": "count",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["metrics"] == ["count"]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_string_point_radius_fixed(monkeypatch):
|
||||
# Legacy deck_scatter with point_radius_fixed as a bare metric key string
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"point_radius_fixed": "count",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["metrics"] == ["count"]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_hex_orderby_when_metrics_present(monkeypatch):
|
||||
# Mirrors BaseDeckGLViz.query_obj(): orderby set from first metric (desc by default)
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
metric = {"expressionType": "SIMPLE", "aggregate": "COUNT", "column": None}
|
||||
form_data = {
|
||||
"viz_type": "deck_hex",
|
||||
"spatial": {"type": "geohash", "geohashCol": "geo"},
|
||||
"size": metric,
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["orderby"] == [(metric, False)]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_scatter_no_orderby_without_metrics(monkeypatch):
|
||||
# No metrics → no orderby (pure spatial column query)
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_scatter",
|
||||
"spatial": {"type": "latlong", "lonCol": "lon", "latCol": "lat"},
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert "orderby" not in queries[0]
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_arc_time_grain(monkeypatch):
|
||||
# deck_arc with time_grain_sqla → is_timeseries, granularity, extras set
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_arc",
|
||||
"spatial": {"type": "latlong", "lonCol": "start_lon", "latCol": "start_lat"},
|
||||
"end_spatial": {
|
||||
"type": "latlong",
|
||||
"lonCol": "end_lon",
|
||||
"latCol": "end_lat",
|
||||
},
|
||||
"granularity_sqla": "ts",
|
||||
"time_grain_sqla": "P1D",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert queries[0]["is_timeseries"] is True
|
||||
assert queries[0]["granularity"] == "ts"
|
||||
assert queries[0].get("extras", {}).get("time_grain_sqla") == "P1D"
|
||||
|
||||
|
||||
def test_build_query_dicts_deck_geojson_ignores_time_grain(monkeypatch):
|
||||
# deck_geojson is not in _DECK_TIMESERIES_VIZ_TYPES; time grain fields not added
|
||||
monkeypatch.setattr(
|
||||
"superset.mcp_service.chart.chart_helpers.resolve_datasource_engine",
|
||||
lambda datasource_id, datasource_type: "base",
|
||||
)
|
||||
form_data = {
|
||||
"viz_type": "deck_geojson",
|
||||
"geojson": "geometry",
|
||||
"granularity_sqla": "ts",
|
||||
"time_grain_sqla": "P1D",
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
|
||||
queries = build_query_dicts_from_form_data(form_data, 1, "table")
|
||||
|
||||
assert "is_timeseries" not in queries[0]
|
||||
assert queries[0].get("extras", {}).get("time_grain_sqla") is None
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from unittest import mock
|
||||
|
||||
from scripts import change_detector
|
||||
|
||||
REPO = "apache/superset"
|
||||
|
||||
|
||||
def test_resolve_workflow_run_files_pull_request(monkeypatch) -> None:
|
||||
"""A workflow_run originating from a pull_request resolves the PR diff."""
|
||||
monkeypatch.setenv("WF_RUN_EVENT", "pull_request")
|
||||
monkeypatch.setenv("WF_RUN_PR_NUMBER", "123")
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
change_detector,
|
||||
"fetch_changed_files_pr",
|
||||
return_value=["superset/foo.py"],
|
||||
) as fetch_pr,
|
||||
mock.patch.object(change_detector, "fetch_changed_files_push") as fetch_push,
|
||||
):
|
||||
files = change_detector.resolve_workflow_run_files(REPO, "deadbeef")
|
||||
|
||||
assert files == ["superset/foo.py"]
|
||||
fetch_pr.assert_called_once_with(REPO, "123")
|
||||
fetch_push.assert_not_called()
|
||||
|
||||
|
||||
def test_resolve_workflow_run_files_push(monkeypatch) -> None:
|
||||
"""A workflow_run originating from a push resolves the push diff via head SHA."""
|
||||
monkeypatch.setenv("WF_RUN_EVENT", "push")
|
||||
monkeypatch.setenv("WF_RUN_HEAD_SHA", "abc123")
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
change_detector,
|
||||
"fetch_changed_files_push",
|
||||
return_value=["superset-frontend/bar.tsx"],
|
||||
) as fetch_push,
|
||||
mock.patch.object(change_detector, "fetch_changed_files_pr") as fetch_pr,
|
||||
):
|
||||
files = change_detector.resolve_workflow_run_files(REPO, "fallback-sha")
|
||||
|
||||
assert files == ["superset-frontend/bar.tsx"]
|
||||
# The originating head SHA wins over the (default-branch) fallback SHA.
|
||||
fetch_push.assert_called_once_with(REPO, "abc123")
|
||||
fetch_pr.assert_not_called()
|
||||
|
||||
|
||||
def test_resolve_workflow_run_files_push_defaults_to_fallback_sha(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Without WF_RUN_HEAD_SHA the push path falls back to the passed SHA."""
|
||||
monkeypatch.delenv("WF_RUN_EVENT", raising=False)
|
||||
monkeypatch.delenv("WF_RUN_HEAD_SHA", raising=False)
|
||||
|
||||
with mock.patch.object(
|
||||
change_detector,
|
||||
"fetch_changed_files_push",
|
||||
return_value=[],
|
||||
) as fetch_push:
|
||||
change_detector.resolve_workflow_run_files(REPO, "fallback-sha")
|
||||
|
||||
fetch_push.assert_called_once_with(REPO, "fallback-sha")
|
||||
|
||||
|
||||
def test_resolve_workflow_run_files_pr_missing_number(monkeypatch) -> None:
|
||||
"""A fork PR without a resolvable number returns None (assume all changed)."""
|
||||
monkeypatch.setenv("WF_RUN_EVENT", "pull_request")
|
||||
monkeypatch.delenv("WF_RUN_PR_NUMBER", raising=False)
|
||||
|
||||
with mock.patch.object(change_detector, "fetch_changed_files_pr") as fetch_pr:
|
||||
files = change_detector.resolve_workflow_run_files(REPO, "deadbeef")
|
||||
|
||||
assert files is None
|
||||
fetch_pr.assert_not_called()
|
||||
|
||||
|
||||
def test_resolve_workflow_run_files_pr_invalid_number(monkeypatch) -> None:
|
||||
"""A non-integer PR number is treated as unresolvable and returns None."""
|
||||
monkeypatch.setenv("WF_RUN_EVENT", "pull_request")
|
||||
monkeypatch.setenv("WF_RUN_PR_NUMBER", "not-a-number")
|
||||
|
||||
with mock.patch.object(change_detector, "fetch_changed_files_pr") as fetch_pr:
|
||||
files = change_detector.resolve_workflow_run_files(REPO, "deadbeef")
|
||||
|
||||
assert files is None
|
||||
fetch_pr.assert_not_called()
|
||||
@@ -0,0 +1,179 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Tests for ``scripts/translations/check_translation_regression.py``.
|
||||
|
||||
The script is not installed as a package, so it is loaded via importlib from
|
||||
its on-disk path.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json # noqa: TID251 - testing a standalone script that uses stdlib json
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
_SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parents[4]
|
||||
/ "scripts"
|
||||
/ "translations"
|
||||
/ "check_translation_regression.py"
|
||||
)
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"check_translation_regression", _SCRIPT_PATH
|
||||
)
|
||||
assert _spec is not None, f"Could not load {_SCRIPT_PATH}"
|
||||
assert _spec.loader is not None, f"No loader on spec for {_SCRIPT_PATH}"
|
||||
check_translation_regression = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(check_translation_regression)
|
||||
|
||||
|
||||
def _compare(
|
||||
tmp_path: Path,
|
||||
before: Mapping[str, object],
|
||||
after: Mapping[str, object],
|
||||
failures: set[str] | None = None,
|
||||
) -> None:
|
||||
"""Run cmd_compare with a baseline file and a mocked 'after' state.
|
||||
|
||||
``failures`` simulates languages whose .po file was present but could not
|
||||
be counted; cmd_compare passes a ``failures`` set into get_counts, so the
|
||||
mock populates it (mirroring the real get_counts contract).
|
||||
"""
|
||||
|
||||
def fake_get_counts(
|
||||
_dir: Path, failures: set[str] | None = None
|
||||
) -> Mapping[str, object]:
|
||||
if failures is not None and _simulated_failures:
|
||||
failures.update(_simulated_failures)
|
||||
return after
|
||||
|
||||
_simulated_failures = failures or set()
|
||||
before_file = tmp_path / "before.json"
|
||||
before_file.write_text(json.dumps(before), encoding="utf-8")
|
||||
with patch.object(
|
||||
check_translation_regression, "get_counts", side_effect=fake_get_counts
|
||||
):
|
||||
check_translation_regression.cmd_compare(str(before_file), tmp_path)
|
||||
|
||||
|
||||
def test_deleting_a_translated_string_is_not_a_regression(tmp_path: Path) -> None:
|
||||
# Translated count drops by 1 (string removed from source) but no new
|
||||
# fuzzy is introduced -> intentional deletion, must NOT be flagged.
|
||||
before = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
after = {"fr": {"translated": 99, "fuzzy": 5}}
|
||||
_compare(tmp_path, before, after) # no SystemExit
|
||||
|
||||
|
||||
def test_renaming_a_string_flags_a_regression(tmp_path: Path) -> None:
|
||||
# A reworded source string strands its translation as fuzzy: translated
|
||||
# drops by 1 AND fuzzy rises by 1 -> real regression.
|
||||
before = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
after = {"fr": {"translated": 99, "fuzzy": 6}}
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
_compare(tmp_path, before, after)
|
||||
assert exc.value.code == 1
|
||||
|
||||
|
||||
def test_no_change_is_clean(tmp_path: Path) -> None:
|
||||
stats = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
_compare(tmp_path, stats, dict(stats)) # no SystemExit
|
||||
|
||||
|
||||
def test_adding_strings_does_not_offset_a_regression(tmp_path: Path) -> None:
|
||||
# Even when the PR adds new (untranslated) strings, a fresh fuzzy elsewhere
|
||||
# is still a regression — additions must not mask it.
|
||||
before = {"de": {"translated": 200, "fuzzy": 10}}
|
||||
after = {"de": {"translated": 205, "fuzzy": 11}}
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
_compare(tmp_path, before, after)
|
||||
assert exc.value.code == 1
|
||||
|
||||
|
||||
def test_legacy_integer_baseline_is_tolerated(tmp_path: Path) -> None:
|
||||
# Older baselines stored a bare translated count with no fuzzy data.
|
||||
before = {"fr": 100}
|
||||
after = {"fr": {"translated": 90, "fuzzy": 0}}
|
||||
_compare(tmp_path, before, after) # deletion-only, no SystemExit
|
||||
|
||||
|
||||
def test_deleting_an_entire_catalog_is_not_a_regression(tmp_path: Path) -> None:
|
||||
# The whole `fr` catalog was intentionally removed: it is absent from
|
||||
# `after` and did NOT fail to count -> a legitimate deletion, not flagged.
|
||||
before = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
after: dict[str, object] = {}
|
||||
_compare(tmp_path, before, after) # no SystemExit
|
||||
|
||||
|
||||
def test_uncountable_baseline_catalog_is_a_hard_failure(tmp_path: Path) -> None:
|
||||
# `fr` still exists on disk but msgfmt could not count it (malformed .po),
|
||||
# so it is missing from `after` AND reported as a failure. This must NOT be
|
||||
# mistaken for an intentional deletion — it is a hard error.
|
||||
before = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
after: dict[str, object] = {}
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
_compare(tmp_path, before, after, failures={"fr"})
|
||||
assert exc.value.code == 1
|
||||
|
||||
|
||||
def test_uncountable_new_catalog_not_in_baseline_is_ignored(tmp_path: Path) -> None:
|
||||
# A catalog that fails to count but was not in the baseline can't be a
|
||||
# regression of a previously-good translation, so it does not fail the run.
|
||||
before = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
after = {"fr": {"translated": 100, "fuzzy": 5}}
|
||||
_compare(tmp_path, before, after, failures={"zz"}) # no SystemExit
|
||||
|
||||
|
||||
def test_writes_report_on_regression(tmp_path: Path) -> None:
|
||||
before_file = tmp_path / "before.json"
|
||||
before_file.write_text(json.dumps({"ja": {"translated": 50, "fuzzy": 2}}))
|
||||
report = tmp_path / "report.md"
|
||||
after = {"ja": {"translated": 49, "fuzzy": 3}}
|
||||
with patch.object(check_translation_regression, "get_counts", return_value=after):
|
||||
with pytest.raises(SystemExit):
|
||||
check_translation_regression.cmd_compare(
|
||||
str(before_file), tmp_path, str(report)
|
||||
)
|
||||
body = report.read_text(encoding="utf-8")
|
||||
assert "Translation Regression Detected" in body
|
||||
assert "`ja`" in body
|
||||
# The report must make clear deletions are not flagged.
|
||||
assert "deleting" in body.lower()
|
||||
|
||||
|
||||
def test_count_stats_parses_translated_and_fuzzy() -> None:
|
||||
stderr = (
|
||||
"3731 translated messages, 1009 fuzzy translations, 175 untranslated messages."
|
||||
)
|
||||
mock_result = MagicMock(stderr=stderr)
|
||||
with patch.object(
|
||||
check_translation_regression.subprocess, "run", return_value=mock_result
|
||||
):
|
||||
stats = check_translation_regression.count_stats(Path("dummy.po"))
|
||||
assert stats == {"translated": 3731, "fuzzy": 1009}
|
||||
|
||||
|
||||
def test_count_stats_defaults_fuzzy_to_zero_when_absent() -> None:
|
||||
# msgfmt omits the fuzzy clause entirely when there are none.
|
||||
mock_result = MagicMock(stderr="42 translated messages.")
|
||||
with patch.object(
|
||||
check_translation_regression.subprocess, "run", return_value=mock_result
|
||||
):
|
||||
stats = check_translation_regression.count_stats(Path("dummy.po"))
|
||||
assert stats == {"translated": 42, "fuzzy": 0}
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
@@ -0,0 +1,162 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Smoke tests for the gevent gunicorn worker.
|
||||
|
||||
Superset is commonly deployed under gunicorn with the gevent worker class
|
||||
(``SERVER_WORKER_CLASS=gevent`` in ``docker/entrypoints/run-server.sh``). The
|
||||
gevent worker monkey-patches the standard library and relies on gevent
|
||||
internals, so a major gevent upgrade can break the worker at boot/serve time in
|
||||
ways that import-only checks do not catch. These tests boot the worker in a
|
||||
subprocess and exercise it, giving CI a real compatibility signal for gevent
|
||||
bumps.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
# These run wherever gevent + gunicorn are installed (e.g. the unit-test job,
|
||||
# which installs requirements/development.txt). Skip cleanly otherwise.
|
||||
pytest.importorskip("gevent")
|
||||
pytest.importorskip("gunicorn")
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
sys.platform.startswith("win"), reason="gunicorn does not run on Windows"
|
||||
)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
def _gunicorn_command() -> list[str]:
|
||||
if gunicorn_bin := shutil.which("gunicorn"):
|
||||
return [gunicorn_bin]
|
||||
# Fall back to the module entry point if the console script isn't on PATH.
|
||||
return [sys.executable, "-m", "gunicorn.app.wsgiapp"]
|
||||
|
||||
|
||||
def test_gunicorn_gevent_worker_serves_request(tmp_path: Path) -> None:
|
||||
"""gunicorn's gevent worker boots and serves an HTTP request."""
|
||||
app_module = tmp_path / "wsgi_smoke_app.py"
|
||||
app_module.write_text(
|
||||
dedent(
|
||||
"""
|
||||
def app(environ, start_response):
|
||||
start_response("200 OK", [("Content-Type", "text/plain")])
|
||||
return [b"ok"]
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
port = _free_port()
|
||||
cmd = _gunicorn_command() + [
|
||||
"--worker-class",
|
||||
"gevent",
|
||||
"--workers",
|
||||
"1",
|
||||
"--bind",
|
||||
f"127.0.0.1:{port}",
|
||||
"--pythonpath",
|
||||
str(tmp_path),
|
||||
"--graceful-timeout",
|
||||
"5",
|
||||
"--log-level",
|
||||
"error",
|
||||
"wsgi_smoke_app:app",
|
||||
]
|
||||
proc = subprocess.Popen( # noqa: S603
|
||||
cmd,
|
||||
cwd=str(tmp_path),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
deadline = time.time() + 30
|
||||
status = None
|
||||
body = None
|
||||
last_err: Exception | None = None
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
output = proc.stdout.read().decode() if proc.stdout else ""
|
||||
pytest.fail(
|
||||
f"gunicorn gevent worker exited early "
|
||||
f"(code {proc.returncode}):\n{output}"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen( # noqa: S310
|
||||
f"http://127.0.0.1:{port}/", timeout=1
|
||||
) as resp:
|
||||
status = resp.status
|
||||
body = resp.read()
|
||||
break
|
||||
except Exception as ex: # noqa: BLE001
|
||||
last_err = ex
|
||||
time.sleep(0.3)
|
||||
else:
|
||||
pytest.fail(
|
||||
f"gunicorn gevent worker did not serve a request in time: {last_err}"
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert body == b"ok"
|
||||
finally:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
def test_gevent_monkey_patch_all_patches_stdlib() -> None:
|
||||
"""gevent's monkey-patching applies cleanly in a fresh interpreter.
|
||||
|
||||
Runs in a subprocess so it never patches the test process, and asserts the
|
||||
core stdlib modules the gevent worker depends on are patched.
|
||||
"""
|
||||
script = dedent(
|
||||
"""
|
||||
from gevent import monkey
|
||||
|
||||
monkey.patch_all()
|
||||
assert monkey.is_module_patched("socket"), "socket not patched"
|
||||
assert monkey.is_module_patched("ssl"), "ssl not patched"
|
||||
print("ok")
|
||||
"""
|
||||
)
|
||||
result = subprocess.run( # noqa: S603
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"gevent monkey.patch_all() failed:\n{result.stdout}\n{result.stderr}"
|
||||
)
|
||||
assert "ok" in result.stdout
|
||||
Reference in New Issue
Block a user