Compare commits

..

7 Commits

Author SHA1 Message Date
Beto Dealmeida
1dfe73d19c Fix tests 2025-08-26 18:10:46 -04:00
Beto Dealmeida
bbda5e2008 Fix tests 2025-08-26 16:22:44 -04:00
Beto Dealmeida
53999c12dd Use Result instead 2025-08-26 12:49:27 -04:00
Beto Dealmeida
f554036d29 Fix tests 2025-08-26 11:06:54 -04:00
Beto Dealmeida
33e7932491 More methods 2025-08-25 18:15:43 -04:00
Beto Dealmeida
92b02d993b More methods 2025-08-25 17:40:31 -04:00
Beto Dealmeida
72ba972e42 chore: standardize DB engine spec query execution 2025-08-25 17:31:15 -04:00
336 changed files with 2970 additions and 11840 deletions

View File

@@ -1,5 +1,5 @@
# Keep this in sync with the base image in the main Dockerfile (ARG PY_VER)
FROM python:3.11.13-trixie AS base
FROM python:3.11.13-bookworm AS base
# Install system dependencies that Superset needs
# This layer will be cached across Codespace sessions

2
.github/CODEOWNERS vendored
View File

@@ -34,7 +34,7 @@
# Notify PMC members of changes to extension-related files
/superset-core/ @michael-s-molina @villebro
/superset-extensions-cli/ @michael-s-molina @villebro
/superset-cli/ @michael-s-molina @villebro
/superset/core/ @michael-s-molina @villebro
/superset/extensions/ @michael-s-molina @villebro
/superset-frontend/src/packages/superset-core/ @michael-s-molina @villebro

View File

@@ -17,9 +17,9 @@ outputs:
docs:
description: Whether docs-related files were changed
value: ${{ steps.change-detector.outputs.docs }}
superset-extensions-cli:
description: Whether superset-extensions-cli package-related files were changed
value: ${{ steps.change-detector.outputs.superset-extensions-cli }}
superset-cli:
description: Whether superset-cli package-related files were changed
value: ${{ steps.change-detector.outputs.superset-cli }}
runs:
using: composite
steps:

View File

@@ -32,7 +32,7 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: true
ref: master

View File

@@ -31,7 +31,7 @@ jobs:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
if: steps.check_queued.outputs.count >= 20
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Cancel duplicate workflow runs
if: steps.check_queued.outputs.count >= 20

View File

@@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -25,7 +25,7 @@ jobs:
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Check and notify
uses: actions/github-script@v7
with:

View File

@@ -71,7 +71,7 @@ jobs:
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
fetch-depth: 1

View File

@@ -31,7 +31,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Check for file changes
id: check

View File

@@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout Repository"
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: "Dependency Review"
uses: actions/dependency-review-action@v4
continue-on-error: true
@@ -53,7 +53,7 @@ jobs:
runs-on: ubuntu-22.04
steps:
- name: "Checkout Repository"
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Setup Python
uses: ./.github/actions/setup-backend/

View File

@@ -22,7 +22,7 @@ jobs:
steps:
- id: set_matrix
run: |
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev", "lean"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize", "py311", "py312"]'; fi)
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev", "lean"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize", "py311"]'; fi)
echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
echo $GITHUB_OUTPUT
@@ -42,7 +42,7 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
@@ -117,7 +117,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check for file changes

View File

@@ -28,7 +28,7 @@ jobs:
run:
working-directory: superset-embedded-sdk
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version-file: './superset-embedded-sdk/.nvmrc'

View File

@@ -18,7 +18,7 @@ jobs:
run:
working-directory: superset-embedded-sdk
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version-file: './superset-embedded-sdk/.nvmrc'

View File

@@ -1,10 +1,4 @@
name: Cleanup ephemeral envs (PR close) [DEPRECATED]
# ⚠️ DEPRECATION NOTICE ⚠️
# This workflow is deprecated and will be removed in a future version.
# The new Superset Showtime workflow handles cleanup automatically.
# See .github/workflows/showtime.yml and showtime-cleanup.yml for replacements.
# Migration guide: https://github.com/mistercrunch/superset-showtime
name: Cleanup ephemeral envs (PR close)
on:
pull_request_target:
@@ -77,5 +71,5 @@ jobs:
issue_number: ${{ github.event.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '⚠️ **DEPRECATED WORKFLOW** - Ephemeral environment shutdown and build artifacts deleted. Please migrate to the new Superset Showtime system for future PRs.'
body: 'Ephemeral environment shutdown and build artifacts deleted.'
})

View File

@@ -1,12 +1,4 @@
name: Ephemeral env workflow [DEPRECATED]
# ⚠️ DEPRECATION NOTICE ⚠️
# This workflow is deprecated and will be removed in a future version.
# Please use the new Superset Showtime workflow instead:
# - Use label "🎪 trigger-start" instead of "testenv-up"
# - Showtime provides better reliability and easier management
# - See .github/workflows/showtime.yml for the replacement
# - Migration guide: https://github.com/mistercrunch/superset-showtime
name: Ephemeral env workflow
# Example manual trigger:
# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field label_name="testenv-up" --field issue_number=666
@@ -134,11 +126,8 @@ jobs:
throw new Error("Issue number is not available.");
}
const body = `⚠️ **DEPRECATED WORKFLOW** ⚠️\n\n@${user} This workflow is deprecated! Please use the new **Superset Showtime** system instead:\n\n` +
`- Replace "testenv-up" label with "🎪 trigger-start"\n` +
`- Better reliability and easier management\n` +
`- See https://github.com/mistercrunch/superset-showtime for details\n\n` +
`Processing your ephemeral environment request [here](${workflowUrl}). Action: **${action}**.` +
const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}).` +
` Action: **${action}**.` +
` More information on [how to use or configure ephemeral environments]` +
`(https://superset.apache.org/docs/contributing/howtos/#github-ephemeral-environments)`;
@@ -160,7 +149,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
ref: ${{ needs.ephemeral-env-label.outputs.sha }}
persist-credentials: false
@@ -220,7 +209,7 @@ jobs:
pull-requests: write
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v4
with:
persist-credentials: false

View File

@@ -27,12 +27,12 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Setup Java
uses: actions/setup-java@v5
uses: actions/setup-java@v4
with:
distribution: "temurin"
java-version: "11"

View File

@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Checkout Repository
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v4

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false

View File

@@ -12,7 +12,7 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -15,12 +15,12 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Setup Java
uses: actions/setup-java@v5
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '11'

View File

@@ -16,7 +16,7 @@ jobs:
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -21,7 +21,7 @@ jobs:
python-version: ["current", "previous", "next"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -27,7 +27,7 @@ jobs:
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -26,7 +26,7 @@ jobs:
name: Bump version and publish package(s)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v4
with:
# pulls all commits (needed for lerna / semantic release to correctly version)
fetch-depth: 0

View File

@@ -1,50 +0,0 @@
name: 🎪 Showtime Cleanup
# Scheduled cleanup of expired environments
on:
schedule:
- cron: '0 */6 * * *' # Every 6 hours
# Manual trigger for testing
workflow_dispatch:
inputs:
max_age_hours:
description: 'Maximum age in hours before cleanup'
required: false
default: '48'
type: string
# Common environment variables
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.AWS_REGION || 'us-west-2' }}
GITHUB_ORG: ${{ github.repository_owner }}
GITHUB_REPO: ${{ github.event.repository.name }}
jobs:
cleanup-expired:
name: Clean up expired showtime environments
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: Install Superset Showtime
run: pip install superset-showtime
- name: Cleanup expired environments
run: |
MAX_AGE="${{ github.event.inputs.max_age_hours || '48' }}"
# Validate max_age is numeric
if [[ ! "$MAX_AGE" =~ ^[0-9]+$ ]]; then
echo "❌ Invalid max_age_hours format: $MAX_AGE (must be numeric)"
exit 1
fi
echo "Cleaning up environments older than ${MAX_AGE}h"
python -m showtime cleanup --older-than "${MAX_AGE}h"

View File

@@ -1,179 +0,0 @@
name: 🎪 Superset Showtime
# Ultra-simple: just sync on any PR state change
on:
pull_request_target:
types: [labeled, unlabeled, synchronize, closed]
# Manual testing
workflow_dispatch:
inputs:
pr_number:
description: 'PR number to sync'
required: true
type: number
sha:
description: 'Specific SHA to deploy (optional, defaults to latest)'
required: false
type: string
# Common environment variables for all jobs (non-sensitive only)
env:
AWS_REGION: us-west-2
GITHUB_ORG: ${{ github.repository_owner }}
GITHUB_REPO: ${{ github.event.repository.name }}
GITHUB_ACTOR: ${{ github.actor }}
jobs:
sync:
name: 🎪 Sync PR to desired state
runs-on: ubuntu-latest
timeout-minutes: 90
permissions:
contents: read
pull-requests: write
steps:
- name: Security Check - Authorize Maintainers Only
id: auth
uses: actions/github-script@v7
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
script: |
const actor = context.actor;
console.log(`🔍 Checking authorization for ${actor}`);
// Early exit for workflow_dispatch - assume authorized since it's manually triggered
if (context.eventName === 'workflow_dispatch') {
console.log(`✅ Workflow dispatch event - assuming authorized for ${actor}`);
core.setOutput('authorized', 'true');
return;
}
const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: actor
});
console.log(`📊 Permission level for ${actor}: ${permission.permission}`);
const authorized = ['write', 'admin'].includes(permission.permission);
if (!authorized) {
console.log(`🚨 Unauthorized user ${actor} - skipping all operations`);
core.setOutput('authorized', 'false');
return;
}
console.log(`✅ Authorized maintainer: ${actor}`);
core.setOutput('authorized', 'true');
// If this is a synchronize event, check if Showtime is active and set blocked label
if (context.eventName === 'pull_request_target' && context.payload.action === 'synchronize') {
console.log(`🔒 Synchronize event detected - checking if Showtime is active`);
// Check if PR has any circus tent labels (Showtime is in use)
const { data: issue } = await github.rest.issues.get({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number
});
const hasCircusLabels = issue.labels.some(label => label.name.startsWith('🎪 '));
if (hasCircusLabels) {
console.log(`🎪 Circus labels found - setting blocked label to prevent auto-deployment`);
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
labels: ['🎪 🔒 showtime-blocked']
});
console.log(`✅ Blocked label set - Showtime will detect and skip operations`);
} else {
console.log(` No circus labels found - Showtime not in use, skipping block`);
}
}
- name: Install Superset Showtime
if: steps.auth.outputs.authorized == 'true'
run: |
echo "::notice::Maintainer ${{ github.actor }} triggered deploy for PR ${{ github.event.pull_request.number || github.event.inputs.pr_number }}"
pip install --upgrade superset-showtime
showtime version
- name: Check what actions are needed
if: steps.auth.outputs.authorized == 'true'
id: check
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Bulletproof PR number extraction
if [[ -n "${{ github.event.pull_request.number }}" ]]; then
PR_NUM="${{ github.event.pull_request.number }}"
elif [[ -n "${{ github.event.inputs.pr_number }}" ]]; then
PR_NUM="${{ github.event.inputs.pr_number }}"
else
echo "❌ No PR number found in event or inputs"
exit 1
fi
echo "Using PR number: $PR_NUM"
# Run sync check-only with optional SHA override
if [[ -n "${{ github.event.inputs.sha }}" ]]; then
OUTPUT=$(python -m showtime sync $PR_NUM --check-only --sha "${{ github.event.inputs.sha }}")
else
OUTPUT=$(python -m showtime sync $PR_NUM --check-only)
fi
echo "$OUTPUT"
# Extract the outputs we need for conditional steps
BUILD=$(echo "$OUTPUT" | grep "build_needed=" | cut -d'=' -f2)
SYNC=$(echo "$OUTPUT" | grep "sync_needed=" | cut -d'=' -f2)
PR_NUM_OUT=$(echo "$OUTPUT" | grep "pr_number=" | cut -d'=' -f2)
TARGET_SHA=$(echo "$OUTPUT" | grep "target_sha=" | cut -d'=' -f2)
echo "build_needed=$BUILD" >> $GITHUB_OUTPUT
echo "sync_needed=$SYNC" >> $GITHUB_OUTPUT
echo "pr_number=$PR_NUM_OUT" >> $GITHUB_OUTPUT
echo "target_sha=$TARGET_SHA" >> $GITHUB_OUTPUT
- name: Checkout PR code (only if build needed)
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.build_needed == 'true'
uses: actions/checkout@v5
with:
ref: ${{ steps.check.outputs.target_sha }}
persist-credentials: false
- name: Setup Docker Environment (only if build needed)
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.build_needed == 'true'
uses: ./.github/actions/setup-docker
with:
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
build: "true"
install-docker-compose: "false"
- name: Execute sync (handles everything)
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.sync_needed == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
PR_NUM="${{ steps.check.outputs.pr_number }}"
TARGET_SHA="${{ steps.check.outputs.target_sha }}"
if [[ -n "$TARGET_SHA" ]]; then
python -m showtime sync $PR_NUM --sha "$TARGET_SHA"
else
python -m showtime sync $PR_NUM
fi

View File

@@ -37,7 +37,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -51,7 +51,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -30,7 +30,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -1,4 +1,4 @@
name: Superset Extensions CLI Package Tests
name: Superset CLI Package Tests
on:
push:
@@ -14,17 +14,17 @@ concurrency:
cancel-in-progress: true
jobs:
test-superset-extensions-cli-package:
test-superset-cli-package:
runs-on: ubuntu-24.04
strategy:
matrix:
python-version: ["previous", "current", "next"]
defaults:
run:
working-directory: superset-extensions-cli
working-directory: superset-cli
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -36,29 +36,29 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.superset-extensions-cli
if: steps.check.outputs.superset-cli
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
requirements-type: dev
- name: Run pytest with coverage
if: steps.check.outputs.superset-extensions-cli
if: steps.check.outputs.superset-cli
run: |
pytest --cov=superset_extensions_cli --cov-report=xml --cov-report=term-missing --cov-report=html -v --tb=short
pytest --cov=superset_cli --cov-report=xml --cov-report=term-missing --cov-report=html -v --tb=short
- name: Upload coverage reports to Codecov
if: steps.check.outputs.superset-extensions-cli
uses: codecov/codecov-action@v5
if: steps.check.outputs.superset-cli
uses: codecov/codecov-action@v3
with:
file: ./coverage.xml
flags: superset-extensions-cli
name: superset-extensions-cli-coverage
flags: superset-cli
name: superset-cli-coverage
fail_ci_if_error: false
- name: Upload HTML coverage report
if: steps.check.outputs.superset-extensions-cli
if: steps.check.outputs.superset-cli
uses: actions/upload-artifact@v4
with:
name: superset-extensions-cli-coverage-html
name: superset-cli-coverage-html
path: htmlcov/

View File

@@ -31,7 +31,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -41,7 +41,7 @@ jobs:
node-version-file: './docs/.nvmrc'
- name: Setup Python
uses: ./.github/actions/setup-backend/
- uses: actions/setup-java@v5
- uses: actions/setup-java@v4
with:
distribution: 'zulu'
java-version: '21'

View File

@@ -18,7 +18,7 @@ jobs:
name: Link Checking
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v4
# Do not bump this linkinator-action version without opening
# an ASF Infra ticket to allow the new version first!
- uses: JustinBeckwith/linkinator-action@v1.11.0
@@ -56,7 +56,7 @@ jobs:
working-directory: docs
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -69,21 +69,21 @@ jobs:
# Conditional checkout based on context
- name: Checkout for push or pull_request event
if: github.event_name == 'push' || github.event_name == 'pull_request'
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
- name: Checkout using ref (workflow_dispatch)
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ github.event.inputs.ref }}
submodules: recursive
- name: Checkout using PR ID (workflow_dispatch)
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != ''
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge

View File

@@ -23,7 +23,7 @@ jobs:
should-run: ${{ steps.check.outputs.frontend }}
steps:
- name: Checkout Code
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
fetch-depth: 0
@@ -47,7 +47,7 @@ jobs:
git show -s --format=raw HEAD
docker buildx build \
-t $TAG \
--cache-from=type=registry,ref=apache/superset-cache:3.10-slim-trixie \
--cache-from=type=registry,ref=apache/superset-cache:3.10-slim-bookworm \
--target superset-node-ci \
.
@@ -73,7 +73,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
name: docker-image
@@ -101,7 +101,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Download Coverage Artifacts
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
pattern: coverage-artifacts-*
path: coverage/
@@ -127,7 +127,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
name: docker-image
@@ -151,7 +151,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
name: docker-image

View File

@@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -29,7 +29,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref_name }}
persist-credentials: true

View File

@@ -41,7 +41,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -99,7 +99,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -152,7 +152,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -48,7 +48,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -108,7 +108,7 @@ jobs:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -24,7 +24,7 @@ jobs:
PYTHONPATH: ${{ github.workspace }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
@@ -49,7 +49,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

View File

@@ -21,7 +21,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Install dependencies

View File

@@ -38,7 +38,7 @@ jobs:
});
- name: "Checkout ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
persist-credentials: false

View File

@@ -42,12 +42,12 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
build_preset: ["dev", "lean", "py310", "websocket", "dockerize", "py311", "py312"]
build_preset: ["dev", "lean", "py310", "websocket", "dockerize", "py311"]
fail-fast: false
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
fetch-depth: 0
@@ -107,7 +107,7 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v5
uses: actions/checkout@v4
with:
fetch-depth: 0

View File

@@ -27,7 +27,7 @@ jobs:
name: Generate Reports
steps:
- name: Checkout Repository
uses: actions/checkout@v5
uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v4

View File

@@ -12,7 +12,7 @@ jobs:
steps:
- name: Welcome Message
uses: actions/first-interaction@v3
uses: actions/first-interaction@v2
continue-on-error: true
with:
repo-token: ${{ github.token }}

View File

@@ -25,7 +25,7 @@ repos:
- id: mypy
name: mypy (main)
args: [--check-untyped-defs]
exclude: ^superset-extensions-cli/
exclude: ^superset-cli/
additional_dependencies: [
types-simplejson,
types-python-dateutil,
@@ -41,9 +41,9 @@ repos:
types-Markdown,
]
- id: mypy
name: mypy (superset-extensions-cli)
name: mypy (superset-cli)
args: [--check-untyped-defs]
files: ^superset-extensions-cli/
files: ^superset-cli/
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:

View File

@@ -73,7 +73,6 @@ ibm-db2.svg
postgresql.svg
snowflake.svg
ydb.svg
loading.svg
# docs-related
erd.puml

View File

@@ -44,8 +44,4 @@ under the License.
- [4.0.1](./CHANGELOG/4.0.1.md)
- [4.0.2](./CHANGELOG/4.0.2.md)
- [4.1.0](./CHANGELOG/4.1.0.md)
- [4.1.1](./CHANGELOG/4.1.1.md)
- [4.1.2](./CHANGELOG/4.1.2.md)
- [4.1.3](./CHANGELOG/4.1.3.md)
- [4.1.4](./CHANGELOG/4.1.4.md)
- [5.0.0](./CHANGELOG/5.0.0.md)

View File

@@ -1,33 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## Change Log
### 4.1.4 (Thu Jul 24 08:30:04 2025 -0300)
**Database Migrations**
**Features**
**Fixes**
- [#34289](https://github.com/apache/superset/pull/34289) fix: Saved queries list break if one query can't be parsed (@michael-s-molina)
- [#33059](https://github.com/apache/superset/pull/33059) fix: Adds missing __init__ file to commands/logs (@michael-s-molina)
**Others**
- [#32236](https://github.com/apache/superset/pull/32236) chore(deps): bump cryptography from 43.0.3 to 44.0.1 (@dependabot[bot])

View File

@@ -18,7 +18,7 @@
######################################################################
# Node stage to deal with static asset construction
######################################################################
ARG PY_VER=3.11.13-slim-trixie
ARG PY_VER=3.11.13-slim-bookworm
# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
@@ -29,7 +29,7 @@ ARG BUILD_TRANSLATIONS="false"
######################################################################
# superset-node-ci used as a base for building frontend assets and CI
######################################################################
FROM --platform=${BUILDPLATFORM} node:20-trixie-slim AS superset-node-ci
FROM --platform=${BUILDPLATFORM} node:20-bookworm-slim AS superset-node-ci
ARG BUILD_TRANSLATIONS
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
ARG DEV_MODE="false" # Skip frontend build in dev mode
@@ -64,7 +64,7 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
--mount=type=cache,target=/root/.cache \
--mount=type=cache,target=/root/.npm \
if [ "${DEV_MODE}" = "false" ]; then \
if [ "$DEV_MODE" = "false" ]; then \
npm ci; \
else \
echo "Skipping 'npm ci' in dev mode"; \
@@ -80,7 +80,7 @@ FROM superset-node-ci AS superset-node
# Build the frontend if not in dev mode
RUN --mount=type=cache,target=/root/.npm \
if [ "${DEV_MODE}" = "false" ]; then \
if [ "$DEV_MODE" = "false" ]; then \
echo "Running 'npm run ${BUILD_CMD}'"; \
npm run ${BUILD_CMD}; \
else \
@@ -91,10 +91,11 @@ RUN --mount=type=cache,target=/root/.npm \
COPY superset/translations /app/superset/translations
# Build translations if enabled, then cleanup localization files
RUN if [ "${BUILD_TRANSLATIONS}" = "true" ]; then \
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
npm run build-translation; \
fi; \
rm -rf /app/superset/translations/*/*/*.[po,mo];
rm -rf /app/superset/translations/*/*/*.po; \
rm -rf /app/superset/translations/*/*/*.mo;
######################################################################
@@ -105,10 +106,10 @@ FROM python:${PY_VER} AS python-base
ARG SUPERSET_HOME="/app/superset_home"
ENV SUPERSET_HOME=${SUPERSET_HOME}
RUN mkdir -p ${SUPERSET_HOME}
RUN mkdir -p $SUPERSET_HOME
RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
&& chmod -R 1777 ${SUPERSET_HOME} \
&& chown -R superset:superset ${SUPERSET_HOME}
&& chmod -R 1777 $SUPERSET_HOME \
&& chown -R superset:superset $SUPERSET_HOME
# Some bash scripts needed throughout the layers
COPY --chmod=755 docker/*.sh /app/docker/
@@ -133,10 +134,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
. /app/.venv/bin/activate && /app/docker/pip-install.sh --requires-build-essential -r requirements/translations.txt
COPY superset/translations/ /app/translations_mo/
RUN if [ "${BUILD_TRANSLATIONS}" = "true" ]; then \
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
pybabel compile -d /app/translations_mo | true; \
fi; \
rm -f /app/translations_mo/*/*/*.[po,json]
rm -f /app/translations_mo/*/*/*.po; \
rm -f /app/translations_mo/*/*/*.json;
######################################################################
# Python APP common layer
@@ -171,11 +173,11 @@ RUN mkdir -p \
ARG INCLUDE_CHROMIUM="false"
ARG INCLUDE_FIREFOX="false"
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
if [ "${INCLUDE_CHROMIUM}" = "true" ] || [ "${INCLUDE_FIREFOX}" = "true" ]; then \
if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
uv pip install playwright && \
playwright install-deps && \
if [ "${INCLUDE_CHROMIUM}" = "true" ]; then playwright install chromium; fi && \
if [ "${INCLUDE_FIREFOX}" = "true" ]; then playwright install firefox; fi; \
if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \
if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \
else \
echo "Skipping browser installation"; \
fi
@@ -261,7 +263,7 @@ COPY requirements/*.txt requirements/
# Copy local packages needed for editable installs in development.txt
COPY superset-core superset-core
COPY superset-extensions-cli superset-extensions-cli
COPY superset-cli superset-cli
# Install Python dependencies using docker/pip-install.sh
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \

View File

@@ -91,7 +91,7 @@ js-format:
cd superset-frontend; npm run prettier
flask-app:
flask run -p 8088 --reload --debugger
flask run -p 8088 --with-threads --reload --debugger
node-app:
cd superset-frontend; npm run dev-server

View File

@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.10-slim-trixie
FROM python:3.10-slim-bookworm
RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset

View File

@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.10-slim-trixie
FROM python:3.10-slim-bookworm
RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset

View File

@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.10-slim-trixie
FROM python:3.10-slim-bookworm
ARG VERSION
RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/superset.git /superset

View File

@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.10-slim-trixie
FROM python:3.10-slim-bookworm
RUN apt-get update -y
RUN apt-get install -y \

View File

@@ -469,10 +469,6 @@ an account first if you don't have one, and reference your username
while requesting access to push packages.
```bash
# Run this first to make sure you are uploading the right version.
# Pypi does not allow you to delete or retract once uplaoded.
twine check dist/*
twine upload dist/*
```
@@ -522,8 +518,6 @@ takes the version (ie `3.1.1`), the git reference (any SHA, tag or branch
reference), and whether to force the `latest` Docker tag on the
generated images.
**NOTE:** If the docker image isn't built, you'll need to run this [GH action](https://github.com/apache/superset/actions/workflows/tag-release.yml) where you provide it the tag sha.
### Npm Release
You might want to publish the latest @superset-ui release to npm

View File

@@ -1,348 +0,0 @@
# Query Sidecar Service Integration
This document describes the Node.js Query Sidecar Service integration that eliminates stale QueryObject issues in Superset Alerts & Reports.
## Problem Statement
Previously, Superset stored QueryObjects in the database after chart visualization logic transformed `form_data` into QueryObject format. This approach had a critical flaw: when the JavaScript visualization code changed, the stored QueryObjects became stale, causing Alerts & Reports to use outdated query logic.
## Solution
The Query Sidecar Service provides a Node.js service that computes QueryObjects from `form_data` on-demand using the same logic as the frontend, ensuring:
- **No stale data**: QueryObjects are computed fresh every time
- **Consistency**: Uses identical logic to the Superset frontend
- **Backward compatibility**: Falls back to legacy screenshot method if sidecar is unavailable
## Architecture
```mermaid
graph TB
A[Superset Frontend] --> B[form_data]
B --> C[Chart Database Record]
D[Alerts & Reports] --> E{Sidecar Available?}
E -->|Yes| F[Query Sidecar Service]
E -->|No| G[Legacy Screenshot Method]
F --> H[buildQueryObject.ts Logic]
H --> I[Fresh QueryObject]
C --> F
I --> J[Chart Data API]
G --> J
```
## Components
### 1. Node.js Sidecar Service (`sidecar-node/`)
Located in `sidecar-node/`, this service provides:
- **REST API**: `POST /api/v1/query-object` to transform form_data
- **Type Safety**: Full TypeScript implementation with Superset type definitions
- **Frontend Compatibility**: Uses identical logic from `superset-ui-core`
- **Health Checks**: `/health` endpoint for monitoring
- **Docker Support**: Production-ready containerization
Key files:
- `src/query/buildQueryObject.ts` - Main transformation logic
- `src/types/index.ts` - TypeScript type definitions
- `src/routes/queryObject.ts` - REST API endpoints
- `Dockerfile` - Container configuration
### 2. Python Client (`superset/utils/query_sidecar.py`)
Python client library that:
- **HTTP Communication**: Handles requests to the sidecar service
- **Error Handling**: Robust error handling with fallback mechanisms
- **QueryObject Creation**: Converts responses to Superset QueryObject instances
- **Configuration**: Configurable timeouts and URLs
### 3. Reports Integration (`superset/commands/report/execute.py`)
Updated report execution logic:
- **Primary Method**: Uses sidecar service to generate fresh QueryObjects
- **Fallback Method**: Falls back to legacy screenshot method on failure
- **Configuration**: Controlled by `QUERY_SIDECAR_ENABLED` setting
## Setup and Configuration
### 1. Deploy the Sidecar Service
#### Development
```bash
cd sidecar-node
npm install
npm run dev
```
#### Production with Docker
```bash
cd sidecar-node
docker build -t superset-query-sidecar .
docker run -p 3001:3001 superset-query-sidecar
```
#### Production with Docker Compose
```bash
docker-compose -f docker-compose.sidecar.yml up -d
```
### 2. Configure Superset
Add to your Superset configuration:
```python
# Enable sidecar service integration
QUERY_SIDECAR_ENABLED = True
# Sidecar service URL
QUERY_SIDECAR_BASE_URL = "http://localhost:3001"
# Request timeout (seconds)
QUERY_SIDECAR_TIMEOUT = 10
```
#### Production Configuration Example
```python
QUERY_SIDECAR_ENABLED = True
QUERY_SIDECAR_BASE_URL = "http://superset-query-sidecar:3001"
QUERY_SIDECAR_TIMEOUT = 30
```
### 3. Verify Integration
#### Check Sidecar Health
```bash
curl http://localhost:3001/health
```
#### Test API Endpoint
```bash
curl -X POST http://localhost:3001/api/v1/query-object \
-H "Content-Type: application/json" \
-d '{
"form_data": {
"datasource": "1__table",
"viz_type": "table",
"metrics": ["count"],
"columns": ["name"]
}
}'
```
## API Reference
### POST /api/v1/query-object
Transforms `form_data` into a QueryObject.
**Request:**
```json
{
"form_data": {
"datasource": "1__table",
"viz_type": "table",
"metrics": ["count"],
"columns": ["name"],
"time_range": "No filter"
},
"query_fields": {
"x": "columns",
"y": "metrics"
}
}
```
**Response:**
```json
{
"query_object": {
"metrics": ["count"],
"columns": ["name"],
"time_range": "No filter",
"filters": [],
"extras": {},
"row_limit": undefined,
"order_desc": true
}
}
```
**Error Response:**
```json
{
"error": "form_data must include datasource and viz_type"
}
```
### GET /health
Returns service health status.
**Response:**
```json
{
"status": "healthy",
"timestamp": "2023-12-01T12:00:00.000Z",
"version": "1.0.0"
}
```
## Error Handling
The integration includes comprehensive error handling:
### Sidecar Service Errors
- **Connection Errors**: Falls back to legacy screenshot method
- **Timeout Errors**: Configurable timeout with fallback
- **Service Errors**: Logs errors and uses fallback method
### Configuration
```python
# Disable sidecar to use legacy method only
QUERY_SIDECAR_ENABLED = False
# Increase timeout for slow networks
QUERY_SIDECAR_TIMEOUT = 30
```
## Migration Strategy
### Phase 1: Parallel Running
1. Deploy sidecar service alongside existing Superset
2. Enable `QUERY_SIDECAR_ENABLED = True`
3. Monitor logs for any fallback usage
### Phase 2: Validation
1. Compare QueryObjects generated by sidecar vs stored versions
2. Verify Alerts & Reports work correctly
3. Monitor performance metrics
### Phase 3: Full Migration
1. Remove dependency on stored query_context (future)
2. Optimize sidecar performance
3. Scale sidecar service as needed
## Testing
### Node.js Service Tests
```bash
cd sidecar-node
npm test
npm run test:watch
```
### Python Integration Tests
```bash
pytest tests/unit_tests/utils/test_query_sidecar.py
pytest tests/unit_tests/commands/report/test_execute_sidecar.py
```
### Integration Testing
1. Create test alerts/reports
2. Verify they execute with fresh QueryObjects
3. Test fallback behavior by stopping sidecar service
## Monitoring and Logging
### Service Monitoring
- Health check endpoint: `GET /health`
- Docker health checks included
- Prometheus metrics (future enhancement)
### Superset Logging
The integration adds detailed logging:
```python
logger.info("Successfully generated query context via sidecar for chart %s", chart_id)
logger.warning("Failed to generate query context via sidecar service: %s. Falling back to screenshot method.", error)
```
### Log Levels
- `INFO`: Successful sidecar operations
- `WARNING`: Fallback to legacy method
- `ERROR`: Critical failures
## Performance Considerations
### Latency
- Sidecar adds ~10-50ms per request
- Network latency between services
- Configurable timeout prevents blocking
### Scaling
- Sidecar service is stateless and can be horizontally scaled
- Consider load balancing for high-volume deployments
- Cache QueryObjects if needed (future enhancement)
### Resource Usage
- Node.js service: ~50-100MB RAM
- CPU usage minimal for typical workloads
- Network bandwidth: ~1-10KB per request
## Future Enhancements
1. **Caching**: Add Redis/Memcached for QueryObject caching
2. **Metrics**: Prometheus metrics for monitoring
3. **Load Balancing**: Support multiple sidecar instances
4. **Query Optimization**: Optimize query generation performance
5. **Database Cleanup**: Remove stored query_context column (breaking change)
## Troubleshooting
### Common Issues
#### Sidecar Service Not Starting
```bash
# Check logs
docker logs superset-query-sidecar
# Verify port availability
netstat -tlnp | grep 3001
```
#### Connection Errors
```bash
# Test connectivity
curl http://localhost:3001/health
# Check Superset logs
grep -i "sidecar" /var/log/superset/superset.log
```
#### Fallback Behavior
```bash
# Check if sidecar is being bypassed
grep -i "falling back" /var/log/superset/superset.log
```
### Configuration Issues
- Verify `QUERY_SIDECAR_BASE_URL` is correct
- Check network connectivity between services
- Ensure sidecar service is healthy
### Performance Issues
- Increase `QUERY_SIDECAR_TIMEOUT` if needed
- Monitor sidecar service resource usage
- Consider scaling sidecar service
## Security Considerations
### Network Security
- Run sidecar service on internal network
- Use HTTPS in production environments
- Configure proper firewall rules
### Input Validation
- Sidecar service validates all inputs
- Superset client validates responses
- No raw SQL execution in sidecar service
### Access Control
- Service-to-service authentication (future)
- Network-level access controls
- Audit logging of sidecar requests

View File

@@ -23,7 +23,6 @@ This file documents any backwards-incompatible changes in Superset and
assists people when migrating to a new version.
## Next
- [34871](https://github.com/apache/superset/pull/34871): Fixed Jest test hanging issue from Ant Design v5 upgrade. MessageChannel is now mocked in test environment to prevent rc-overflow from causing Jest to hang. Test environment only - no production impact.
- [34782](https://github.com/apache/superset/pull/34782): Dataset exports now include the dataset ID in their file name (similar to charts and dashboards). If managing assets as code, make sure to rename existing dataset YAMLs to include the ID (and avoid duplicated files).
- [34536](https://github.com/apache/superset/pull/34536): The `ENVIRONMENT_TAG_CONFIG` color values have changed to support only Ant Design semantic colors. Update your `superset_config.py`:
- Change `"error.base"` to just `"error"` after this PR

View File

@@ -71,7 +71,7 @@ x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
cache_from:
- apache/superset-cache:3.10-slim-trixie
- apache/superset-cache:3.10-slim-bookworm
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
@@ -118,8 +118,6 @@ services:
POSTGRES_DB: superset_light
SUPERSET__SQLALCHEMY_EXAMPLES_URI: "duckdb:////app/data/examples.duckdb"
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
GITHUB_HEAD_REF: ${GITHUB_HEAD_REF:-}
GITHUB_SHA: ${GITHUB_SHA:-}
superset-init-light:
build:

View File

@@ -33,7 +33,7 @@ x-common-build: &common-build
context: .
target: dev
cache_from:
- apache/superset-cache:3.10-slim-trixie
- apache/superset-cache:3.10-slim-bookworm
services:
redis:

View File

@@ -1,60 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
version: '3.8'
services:
superset-query-sidecar:
build:
context: ./sidecar-node
dockerfile: Dockerfile
container_name: superset-query-sidecar
ports:
- "3001:3001"
environment:
- NODE_ENV=production
- HOST=0.0.0.0
- PORT=3001
- SUPERSET_ORIGINS=http://localhost:8088,http://superset:8088
healthcheck:
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3001/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => process.exit(1))"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
restart: unless-stopped
networks:
- superset
# Example integration with existing Superset service
# Uncomment and modify according to your setup
#
# superset:
# # ... your existing superset configuration
# environment:
# - QUERY_SIDECAR_ENABLED=true
# - QUERY_SIDECAR_BASE_URL=http://superset-query-sidecar:3001
# - QUERY_SIDECAR_TIMEOUT=30
# depends_on:
# superset-query-sidecar:
# condition: service_healthy
# networks:
# - superset
networks:
superset:
external: true

View File

@@ -36,7 +36,7 @@ x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
cache_from:
- apache/superset-cache:3.10-slim-trixie
- apache/superset-cache:3.10-slim-bookworm
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}

View File

@@ -18,7 +18,7 @@
set -euo pipefail
# Ensure this script is run as root
if [[ ${EUID} -ne 0 ]]; then
if [[ $EUID -ne 0 ]]; then
echo "This script must be run as root" >&2
exit 1
fi
@@ -42,7 +42,7 @@ echo -e "${GREEN}Installing packages: $@${RESET}"
apt-get install -yqq --no-install-recommends "$@"
echo -e "${GREEN}Autoremoving unnecessary packages...${RESET}"
apt-get autoremove -yqq --purge
apt-get autoremove -y
echo -e "${GREEN}Cleaning up package cache and metadata...${RESET}"
apt-get clean

View File

@@ -72,7 +72,7 @@ case "${1}" in
;;
app)
echo "Starting web app (using development server)..."
flask run -p $PORT --reload --debugger --without-threads --host=0.0.0.0
flask run -p $PORT --with-threads --reload --debugger --host=0.0.0.0
;;
app-gunicorn)
echo "Starting web app..."

View File

@@ -38,14 +38,14 @@ for arg in "$@"; do
done
# Install build-essential if required
if ${REQUIRES_BUILD_ESSENTIAL}; then
if $REQUIRES_BUILD_ESSENTIAL; then
echo "Installing build-essential for package builds..."
apt-get update -qq \
&& apt-get install -yqq --no-install-recommends build-essential
fi
# Choose whether to use pip cache
if ${USE_CACHE}; then
if $USE_CACHE; then
echo "Using pip cache..."
uv pip install "${ARGS[@]}"
else
@@ -54,7 +54,7 @@ else
fi
# Remove build-essential if it was installed
if ${REQUIRES_BUILD_ESSENTIAL}; then
if $REQUIRES_BUILD_ESSENTIAL; then
echo "Removing build-essential to keep the image lean..."
apt-get autoremove -yqq --purge build-essential \
&& apt-get clean \

View File

@@ -12,7 +12,7 @@ Users can configure automated alerts and reports to send dashboards or charts to
- *Alerts* are sent when a SQL condition is reached
- *Reports* are sent on a schedule
Alerts and reports are disabled by default. To turn them on, you'll need to change configuration settings and install a suitable headless browser in your environment.
Alerts and reports are disabled by default. To turn them on, you need to do some setup, described here.
## Requirements
@@ -35,14 +35,16 @@ Screenshots will be taken but no messages actually sent as long as `ALERT_REPORT
#### In your `Dockerfile`
You'll need to extend the Superset image to include a headless browser. Your options include:
- Use Playwright with Chrome: this is the recommended approach as of version >=4.1.x. A working example of a Dockerfile that installs these tools is provided under “Building your own production Docker image” on the [Docker Builds](/docs/installation/docker-builds#building-your-own-production-docker-image) page. Read the code comments there as you'll also need to change a feature flag in your config.
- Use Firefox: you'll need to install geckodriver and Firefox.
- Use Chrome without Playwright: you'll need to install Chrome and set the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`.
- You must install a headless browser, for taking screenshots of the charts and dashboards. Only Firefox and Chrome are currently supported.
> If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`.
In Superset versions <=4.0x, users installed Firefox or Chrome and that was documented here.
Note: All the components required (Firefox headless browser, Redis, Postgres db, celery worker and celery beat) are present in the *dev* docker image if you are following [Installing Superset Locally](/docs/installation/docker-compose/).
All you need to do is add the required config variables described in this guide (See `Detailed Config`).
Only the worker container needs the browser.
If you are running a non-dev docker image, e.g., a stable release like `apache/superset:3.1.0`, that image does not include a headless browser. Only the `superset_worker` container needs this headless browser to browse to the target chart or dashboard.
You can either install and configure the headless browser - see "Custom Dockerfile" section below - or when deploying via `docker compose`, modify your `docker-compose.yml` file to use a dev image for the worker container and a stable release image for the `superset_app` container.
*Note*: In this context, a "dev image" is the same application software as its corresponding non-dev image, just bundled with additional tools. So an image like `3.1.0-dev` is identical to `3.1.0` when it comes to stability, functionality, and running in production. The actual "in-development" versions of Superset - cutting-edge and unstable - are not tagged with version numbers on Docker Hub and will display version `0.0.0-dev` within the Superset UI.
### Slack integration
@@ -150,8 +152,8 @@ SMTP_MAIL_FROM = "noreply@youremail.com"
EMAIL_REPORTS_SUBJECT_PREFIX = "[Superset] " # optional - overwrites default value in config.py of "[Report] "
# WebDriver configuration
# If you use Firefox or Playwright with Chrome, you can stick with default values
# If you use Chrome and are *not* using Playwright, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS
# If you use Firefox, you can stick with default values
# If you use Chrome, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS
WEBDRIVER_TYPE = "chrome"
WEBDRIVER_OPTION_ARGS = [
"--force-device-scale-factor=2.0",
@@ -217,6 +219,62 @@ def alert_dynamic_minimal_interval(**kwargs) -> int:
ALERT_MINIMUM_INTERVAL = alert_dynamic_minimal_interval
```
## Custom Dockerfile
If you're running the dev version of a released Superset image, like `apache/superset:3.1.0-dev`, you should be set with the above.
But if you're building your own image, or starting with a non-dev version, a webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient.
Here's how you can modify your Dockerfile to take the screenshots either with Firefox or Chrome.
### Using Firefox
```docker
FROM apache/superset:3.1.0
USER root
RUN apt-get update && \
apt-get install --no-install-recommends -y firefox-esr
ENV GECKODRIVER_VERSION=0.29.0
RUN wget -q https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz && \
tar -x geckodriver -zf geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz -O > /usr/bin/geckodriver && \
chmod 755 /usr/bin/geckodriver && \
rm geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz
RUN pip install --no-cache gevent psycopg2 redis
USER superset
```
### Using Chrome
```docker
FROM apache/superset:3.1.0
USER root
RUN apt-get update && \
apt-get install -y wget zip libaio1
RUN export CHROMEDRIVER_VERSION=$(curl --silent https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_116) && \
wget -O google-chrome-stable_current_amd64.deb -q http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROMEDRIVER_VERSION}-1_amd64.deb && \
apt-get install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \
rm -f google-chrome-stable_current_amd64.deb
RUN export CHROMEDRIVER_VERSION=$(curl --silent https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_116) && \
wget -q https://storage.googleapis.com/chrome-for-testing-public/${CHROMEDRIVER_VERSION}/linux64/chromedriver-linux64.zip && \
unzip -j chromedriver-linux64.zip -d /usr/bin && \
chmod 755 /usr/bin/chromedriver && \
rm -f chromedriver-linux64.zip
RUN pip install --no-cache gevent psycopg2 redis
USER superset
```
Don't forget to set `WEBDRIVER_TYPE` and `WEBDRIVER_OPTION_ARGS` in your config if you use Chrome.
## Troubleshooting
There are many reasons that reports might not be working. Try these steps to check for specific issues.
@@ -235,7 +293,9 @@ This is the best source of information about the problem. In a docker compose d
To take a screenshot, the worker visits the dashboard or chart using a headless browser, then takes a screenshot. If you are able to send a chart as CSV or text but can't send as PNG, your problem may lie with the browser.
If you are handling the installation of the headless browser on your own, do your own verification to ensure that the headless browser opens successfully in the worker environment.
Superset docker images that have a tag ending with `-dev` have the Firefox headless browser and geckodriver already installed. You can test that these are installed and in the proper path by entering your Superset worker and running `firefox --headless` and then `geckodriver`. Both commands should start those applications.
If you are handling the installation of that software on your own, or wish to use Chromium instead, do your own verification to ensure that the headless browser opens successfully in the worker environment.
### Send a test email

View File

@@ -747,26 +747,6 @@ To run a single test file:
npm run test -- path/to/file.js
```
#### Known Issues and Workarounds
**Jest Test Hanging (MessageChannel Issue)**
If Jest tests hang with "Jest did not exit one second after the test run has completed", this is likely due to the MessageChannel issue from rc-overflow (Ant Design v5 components).
**Root Cause**: `rc-overflow@1.4.1` creates MessageChannel handles for responsive overflow detection that remain open after test completion.
**Current Workaround**: MessageChannel is mocked as undefined in `spec/helpers/jsDomWithFetchAPI.ts`, forcing rc-overflow to use requestAnimationFrame fallback.
**To verify if still needed**: Remove the MessageChannel mocking lines and run `npm test -- --shard=4/8`. If tests hang, the workaround is still required.
**Future removal conditions**: This workaround can be removed when:
- rc-overflow updates to properly clean up MessagePorts in test environments
- Jest updates to handle MessageChannel/MessagePort cleanup better
- Ant Design switches away from rc-overflow
- We switch away from Ant Design v5
**See**: [PR #34871](https://github.com/apache/superset/pull/34871) for full technical details.
### Debugging Server App
#### Local

View File

@@ -86,6 +86,8 @@ USER root
ENV PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright-browsers
# Install packages using uv into the virtual environment
# Superset started using uv after the 4.1 branch; if you are building from apache/superset:4.1.x or an older version,
# replace the first two lines with RUN pip install \
RUN . /app/.venv/bin/activate && \
uv pip install \
# install psycopg2 for using PostgreSQL metadata store - could be a MySQL package if using that backend:

View File

@@ -35,7 +35,7 @@
"@emotion/core": "^10.0.27",
"@emotion/react": "^11.13.3",
"@emotion/styled": "^10.0.27",
"@mdx-js/react": "^3.1.1",
"@mdx-js/react": "^3.1.0",
"@saucelabs/theme-github-codeblock": "^0.3.0",
"@storybook/addon-docs": "^8.6.11",
"@storybook/blocks": "^8.6.11",
@@ -50,7 +50,7 @@
"@storybook/theming": "^8.6.11",
"@superset-ui/core": "^0.20.4",
"antd": "^5.26.7",
"caniuse-lite": "^1.0.30001739",
"caniuse-lite": "^1.0.30001707",
"docusaurus-plugin-less": "^2.0.2",
"json-bigint": "^1.0.0",
"less": "^4.4.0",
@@ -74,13 +74,13 @@
"@types/react": "^19.1.8",
"@typescript-eslint/eslint-plugin": "^8.37.0",
"@typescript-eslint/parser": "^8.37.0",
"eslint": "^9.34.0",
"eslint": "^9.32.0",
"eslint-config-prettier": "^10.1.8",
"eslint-plugin-prettier": "^5.5.3",
"eslint-plugin-react": "^7.37.5",
"globals": "^16.3.0",
"prettier": "^3.6.2",
"typescript": "~5.9.2",
"typescript": "~5.8.3",
"typescript-eslint": "^8.39.0",
"webpack": "^5.101.0"
},

View File

@@ -962,11 +962,8 @@
"ChartDataDatasource": {
"properties": {
"id": {
"description": "Datasource id/uuid",
"oneOf": [
{ "type": "integer" },
{ "type": "string" }
]
"description": "Datasource id",
"type": "integer"
},
"type": {
"description": "Datasource type",

View File

@@ -2433,10 +2433,10 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
"@eslint/js@9.34.0", "@eslint/js@^9.32.0":
version "9.34.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.34.0.tgz#fc423168b9d10e08dea9088d083788ec6442996b"
integrity sha512-EoyvqQnBNsV1CWaEJ559rxXL4c8V92gxirbawSmVUOWXlsRxxQXl6LmCpdUblgxgSkDIqKnhzba2SjRTI/A5Rw==
"@eslint/js@9.33.0", "@eslint/js@^9.32.0":
version "9.33.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.33.0.tgz#475c92fdddab59b8b8cab960e3de2564a44bf368"
integrity sha512-5K1/mKhWaMfreBGJTwval43JJmkip0RmM+3+IuqupeSKNC/Th2Kc7ucaq5ovTSra/OOKB9c58CGSz3QMVbWt0A==
"@eslint/object-schema@^2.1.6":
version "2.1.6"
@@ -2598,10 +2598,10 @@
unist-util-visit "^5.0.0"
vfile "^6.0.0"
"@mdx-js/react@^3.0.0", "@mdx-js/react@^3.1.1":
version "3.1.1"
resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-3.1.1.tgz#24bda7fffceb2fe256f954482123cda1be5f5fef"
integrity sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==
"@mdx-js/react@^3.0.0", "@mdx-js/react@^3.1.0":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-3.1.0.tgz#c4522e335b3897b9a845db1dbdd2f966ae8fb0ed"
integrity sha512-QjHtSaoameoalGnKDT3FoIl4+9RwyTmo9ZJGBdLOks/YOiWHoRDI3PUwEzOE7kEmGcV3AFcp9K6dYu9rEuKLAQ==
dependencies:
"@types/mdx" "^2.0.0"
@@ -5020,10 +5020,10 @@ caniuse-api@^3.0.0:
lodash.memoize "^4.1.2"
lodash.uniq "^4.5.0"
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001702, caniuse-lite@^1.0.30001735, caniuse-lite@^1.0.30001739:
version "1.0.30001739"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001739.tgz#b34ce2d56bfc22f4352b2af0144102d623a124f4"
integrity sha512-y+j60d6ulelrNSwpPyrHdl+9mJnQzHBr08xm48Qno0nSk4h3Qojh+ziv2qE6rXf4k3tadF4o1J/1tAbVm1NtnA==
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001702, caniuse-lite@^1.0.30001707, caniuse-lite@^1.0.30001735:
version "1.0.30001735"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001735.tgz#ba658fd3fd24a4106fd68d5ce472a2c251494dbe"
integrity sha512-EV/laoX7Wq2J9TQlyIXRxTJqIw4sxfXS4OYgudGxBYRuTv0q7AM6yMEpU/Vo1I94thg9U6EZ2NfZx9GJq83u7w==
ccount@^2.0.0:
version "2.0.1"
@@ -6713,10 +6713,10 @@ eslint-visitor-keys@^4.2.1:
resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz#4cfea60fe7dd0ad8e816e1ed026c1d5251b512c1"
integrity sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==
eslint@^9.34.0:
version "9.34.0"
resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.34.0.tgz#0ea1f2c1b5d1671db8f01aa6b8ce722302016f7b"
integrity sha512-RNCHRX5EwdrESy3Jc9o8ie8Bog+PeYvvSR8sDGoZxNFTvZ4dlxUB3WzQ3bQMztFrSRODGrLLj8g6OFuGY/aiQg==
eslint@^9.32.0:
version "9.33.0"
resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.33.0.tgz#cc186b3d9eb0e914539953d6a178a5b413997b73"
integrity sha512-TS9bTNIryDzStCpJN93aC5VRSW3uTx9sClUn4B87pwiCaJh220otoI0X8mJKr+VcPtniMdN8GKjlwgWGUv5ZKA==
dependencies:
"@eslint-community/eslint-utils" "^4.2.0"
"@eslint-community/regexpp" "^4.12.1"
@@ -6724,7 +6724,7 @@ eslint@^9.34.0:
"@eslint/config-helpers" "^0.3.1"
"@eslint/core" "^0.15.2"
"@eslint/eslintrc" "^3.3.1"
"@eslint/js" "9.34.0"
"@eslint/js" "9.33.0"
"@eslint/plugin-kit" "^0.3.5"
"@humanfs/node" "^0.16.6"
"@humanwhocodes/module-importer" "^1.0.1"
@@ -13269,10 +13269,10 @@ typescript-eslint@^8.39.0:
"@typescript-eslint/typescript-estree" "8.40.0"
"@typescript-eslint/utils" "8.40.0"
typescript@~5.9.2:
version "5.9.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.9.2.tgz#d93450cddec5154a2d5cabe3b8102b83316fb2a6"
integrity sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==
typescript@~5.8.3:
version "5.8.3"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.8.3.tgz#92f8a3e5e3cf497356f4178c34cd65a7f5e8440e"
integrity sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==
ufo@^1.5.4:
version "1.6.1"

View File

@@ -35,8 +35,7 @@ classifiers = [
"Programming Language :: Python :: 3.12",
]
dependencies = [
# no bounds for apache-superset-core until we have a stable version
"apache-superset-core",
"apache-superset-core>=0.0.1, <0.2",
"backoff>=1.8.0",
"celery>=5.3.6, <6.0.0",
"click>=8.0.3",
@@ -48,7 +47,7 @@ dependencies = [
"cryptography>=42.0.4, <45.0.0",
"deprecation>=2.1.0, <2.2.0",
"flask>=2.2.5, <3.0.0",
"flask-appbuilder>=4.8.1, <5.0.0",
"flask-appbuilder>=4.8.0, <5.0.0",
"flask-caching>=2.1.0, <3",
"flask-compress>=1.13, <2.0",
"flask-talisman>=1.0.0, <2.0",
@@ -69,7 +68,6 @@ dependencies = [
"markdown>=3.0",
# marshmallow>=4 has issues: https://github.com/apache/superset/issues/33162
"marshmallow>=3.0, <4",
"marshmallow-union>=0.1",
"msgpack>=1.0.0, <1.1",
"nh3>=0.2.11, <0.3",
"numpy>1.23.5, <2.3",
@@ -124,8 +122,8 @@ cockroachdb = ["cockroachdb>=0.3.5, <0.4"]
crate = ["sqlalchemy-cratedb>=0.40.1, <1"]
databend = ["databend-sqlalchemy>=0.3.2, <1.0"]
databricks = [
"databricks-sql-connector==4.1.2",
"databricks-sqlalchemy==1.0.5",
"databricks-sql-connector>=2.0.2, <3",
"sqlalchemy-databricks>=0.2.0",
]
db2 = ["ibm-db-sa>0.3.8, <=0.4.0"]
denodo = ["denodo-sqlalchemy~=1.0.6"]
@@ -194,8 +192,7 @@ doris = ["pydoris>=1.0.0, <2.0.0"]
oceanbase = ["oceanbase_py>=0.0.1"]
ydb = ["ydb-sqlalchemy>=0.1.2"]
development = [
# no bounds for apache-superset-extensions-cli until a stable version
"apache-superset-extensions-cli",
"apache-superset-cli>=0.0.1, <0.2",
"docker",
"flask-testing",
"freezegun",
@@ -227,8 +224,8 @@ documentation = "https://superset.apache.org/docs/intro"
combine_as_imports = true
include_trailing_comma = true
line_length = 88
known_first_party = "superset, apache-superset-core, apache-superset-extensions-cli"
known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, marshmallow-union, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
known_first_party = "superset, apache-superset-core, apache-superset-cli"
known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
multi_line_output = 3
order_by_type = false
@@ -433,4 +430,4 @@ pyxlsb = "1" # GPL
[tool.uv.sources]
apache-superset-core = { path = "./superset-core", editable = true }
apache-superset-extensions-cli = { path = "./superset-extensions-cli", editable = true }
apache-superset-cli = { path = "./superset-cli", editable = true }

View File

@@ -114,7 +114,7 @@ flask==2.3.3
# flask-session
# flask-sqlalchemy
# flask-wtf
flask-appbuilder==4.8.1
flask-appbuilder==4.8.0
# via
# apache-superset (pyproject.toml)
# apache-superset-core
@@ -219,13 +219,10 @@ marshmallow==3.26.1
# apache-superset (pyproject.toml)
# flask-appbuilder
# marshmallow-sqlalchemy
# marshmallow-union
marshmallow-sqlalchemy==1.4.0
# via
# -r requirements/base.in
# flask-appbuilder
marshmallow-union==0.1.15
# via apache-superset (pyproject.toml)
mdurl==0.1.2
# via markdown-it-py
msgpack==1.0.8

View File

@@ -17,4 +17,4 @@
# under the License.
#
-e .[development,bigquery,druid,duckdb,gevent,gsheets,mysql,postgres,presto,prophet,trino,thumbnails]
-e ./superset-extensions-cli[test]
-e ./superset-cli[test]

View File

@@ -2,14 +2,14 @@
# uv pip compile requirements/development.in -c requirements/base-constraint.txt -o requirements/development.txt
-e .
# via -r requirements/development.in
-e ./superset-core
# via
# apache-superset
# apache-superset-extensions-cli
-e ./superset-extensions-cli
-e ./superset-cli
# via
# -r requirements/development.in
# apache-superset
-e ./superset-core
# via
# apache-superset
# apache-superset-cli
alembic==1.15.2
# via
# -c requirements/base-constraint.txt
@@ -102,7 +102,7 @@ click==8.2.1
# via
# -c requirements/base-constraint.txt
# apache-superset
# apache-superset-extensions-cli
# apache-superset-cli
# celery
# click-didyoumean
# click-option-group
@@ -208,7 +208,7 @@ flask==2.3.3
# flask-sqlalchemy
# flask-testing
# flask-wtf
flask-appbuilder==4.8.1
flask-appbuilder==4.8.0
# via
# -c requirements/base-constraint.txt
# apache-superset
@@ -382,7 +382,7 @@ itsdangerous==2.2.0
jinja2==3.1.6
# via
# -c requirements/base-constraint.txt
# apache-superset-extensions-cli
# apache-superset-cli
# flask
# flask-babel
jsonpath-ng==1.7.0
@@ -444,15 +444,10 @@ marshmallow==3.26.1
# apache-superset
# flask-appbuilder
# marshmallow-sqlalchemy
# marshmallow-union
marshmallow-sqlalchemy==1.4.0
# via
# -c requirements/base-constraint.txt
# flask-appbuilder
marshmallow-union==0.1.15
# via
# -c requirements/base-constraint.txt
# apache-superset
matplotlib==3.9.0
# via prophet
mccabe==0.7.0
@@ -674,17 +669,17 @@ pysocks==1.7.1
pytest==7.4.4
# via
# apache-superset
# apache-superset-extensions-cli
# apache-superset-cli
# pytest-cov
# pytest-mock
pytest-cov==6.0.0
# via
# apache-superset
# apache-superset-extensions-cli
# apache-superset-cli
pytest-mock==3.10.0
# via
# apache-superset
# apache-superset-extensions-cli
# apache-superset-cli
python-dateutil==2.9.0.post0
# via
# -c requirements/base-constraint.txt
@@ -781,7 +776,7 @@ selenium==4.32.0
# -c requirements/base-constraint.txt
# apache-superset
semver==3.0.4
# via apache-superset-extensions-cli
# via apache-superset-cli
setuptools==80.7.1
# via
# nodeenv
@@ -909,7 +904,7 @@ watchdog==6.0.0
# via
# -c requirements/base-constraint.txt
# apache-superset
# apache-superset-extensions-cli
# apache-superset-cli
wcwidth==0.2.13
# via
# -c requirements/base-constraint.txt

View File

@@ -45,9 +45,9 @@ PATTERNS = {
"docs": [
r"^docs/",
],
"superset-extensions-cli": [
r"^\.github/workflows/superset-extensions-cli\.yml",
r"^superset-extensions-cli/",
"superset-cli": [
r"^\.github/workflows/superset-cli\.yml",
r"^superset-cli/",
r"^superset-core/",
],
}

View File

@@ -1,51 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
FROM node:18-alpine
# Set working directory
WORKDIR /app
# Copy package files
COPY package*.json ./
# Install dependencies
RUN npm ci --only=production
# Copy source code
COPY . .
# Build the application
RUN npm run build
# Create non-root user
RUN addgroup -g 1001 -S nodejs && \
adduser -S sidecar -u 1001
# Change ownership of the app directory
RUN chown -R sidecar:nodejs /app
USER sidecar
# Expose port
EXPOSE 3001
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://localhost:3001/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => process.exit(1))"
# Start the application
CMD ["npm", "start"]

View File

@@ -1,143 +0,0 @@
# Superset Query Sidecar Service
A Node.js sidecar service that computes Superset QueryObjects from form_data, eliminating the need to store stale query objects in the database for Alerts & Reports.
## Overview
This service provides a REST API that transforms Superset's `form_data` into `QueryObject` format using the same logic as the frontend, ensuring consistency and eliminating staleness issues in Alerts & Reports.
## Features
- **Real-time QueryObject generation**: No stale data from database
- **Frontend-compatible logic**: Uses the same transformation logic as superset-ui-core
- **TypeScript support**: Full type safety
- **Docker support**: Easy deployment
- **Health checks**: Built-in monitoring endpoints
- **CORS support**: Configurable for Superset integration
## Quick Start
### Development
```bash
# Install dependencies
npm install
# Start development server
npm run dev
# Run tests
npm test
```
### Production
```bash
# Build the application
npm run build
# Start production server
npm start
```
### Docker
```bash
# Build Docker image
docker build -t superset-query-sidecar .
# Run container
docker run -p 3001:3001 superset-query-sidecar
```
## API Reference
### POST /api/v1/query-object
Transforms form_data into a QueryObject.
**Request:**
```json
{
"form_data": {
"datasource": "1__table",
"viz_type": "table",
"metrics": ["count"],
"columns": ["name"],
"time_range": "No filter"
},
"query_fields": {
"x": "columns",
"y": "metrics"
}
}
```
**Response:**
```json
{
"query_object": {
"datasource": "1__table",
"metrics": ["count"],
"columns": ["name"],
"time_range": "No filter",
"filters": [],
"extras": {}
}
}
```
### GET /health
Health check endpoint.
**Response:**
```json
{
"status": "healthy",
"timestamp": "2023-12-01T12:00:00.000Z",
"version": "1.0.0"
}
```
## Configuration
Environment variables:
- `PORT`: Server port (default: 3001)
- `HOST`: Server host (default: localhost)
- `NODE_ENV`: Environment mode (development/production)
- `SUPERSET_ORIGINS`: Allowed CORS origins (comma-separated)
## Integration with Superset
This service is designed to be called by Superset's Python backend to generate QueryObjects for Alerts & Reports, replacing the current approach of reading stale query objects from the database.
## Architecture
```
┌─────────────────┐ ┌──────────────────────┐ ┌─────────────────────┐
│ Superset │ │ Query Sidecar │ │ Alerts & Reports │
│ Frontend │ │ Service │ │ │
│ │ │ │ │ │
│ form_data ────┼────┼──► buildQueryObject │◄───┼─── Python Client │
│ │ │ │ │ │
└─────────────────┘ └──────────────────────┘ └─────────────────────┘
```
## Testing
```bash
# Run all tests
npm test
# Run tests in watch mode
npm run test:watch
# Run tests with coverage
npm test -- --coverage
```
## License
Licensed under the Apache License, Version 2.0.

View File

@@ -1,16 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
roots: ['<rootDir>/src'],
testMatch: ['**/__tests__/**/*.test.ts', '**/?(*.)+(spec|test).ts'],
transform: {
'^.+\\.ts$': 'ts-jest',
},
collectCoverageFrom: [
'src/**/*.ts',
'!src/**/*.d.ts',
'!src/**/*.test.ts',
],
coverageDirectory: 'coverage',
coverageReporters: ['text', 'lcov', 'html'],
};

View File

@@ -1,41 +0,0 @@
{
"name": "superset-query-sidecar",
"version": "1.0.0",
"description": "Node.js sidecar service for computing Superset QueryObjects from form_data",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node src/index.ts",
"test": "jest",
"test:watch": "jest --watch"
},
"dependencies": {
"express": "^4.18.2",
"cors": "^2.8.5",
"helmet": "^7.1.0",
"compression": "^1.7.4"
},
"devDependencies": {
"@types/express": "^4.17.21",
"@types/node": "^20.10.0",
"@types/cors": "^2.8.17",
"@types/compression": "^1.7.5",
"@types/jest": "^29.5.8",
"typescript": "^5.3.2",
"ts-node": "^10.9.1",
"jest": "^29.7.0",
"ts-jest": "^29.1.1"
},
"engines": {
"node": ">=18.0.0"
},
"keywords": [
"superset",
"query",
"sidecar",
"analytics"
],
"author": "Apache Superset",
"license": "Apache-2.0"
}

View File

@@ -1,149 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import buildQueryObject from '../query/buildQueryObject';
import { QueryFormData } from '../types';
describe('buildQueryObject', () => {
const baseFormData: QueryFormData = {
datasource: '1__table',
viz_type: 'table',
time_range: 'No filter',
metrics: ['count'],
columns: ['name', 'category'],
row_limit: 1000,
order_desc: true,
};
it('should build a basic query object', () => {
const queryObject = buildQueryObject(baseFormData);
expect(queryObject).toEqual({
time_range: 'No filter',
since: undefined,
until: undefined,
granularity: undefined,
columns: ['name', 'category'],
metrics: ['count'],
orderby: undefined,
annotation_layers: [],
row_limit: 1000,
row_offset: undefined,
series_columns: undefined,
series_limit: 0,
series_limit_metric: undefined,
group_others_when_limit_reached: false,
order_desc: true,
url_params: undefined,
custom_params: {},
extras: {
filters: [],
},
filters: [],
custom_form_data: {},
});
});
it('should handle adhoc filters', () => {
const formDataWithFilters: QueryFormData = {
...baseFormData,
adhoc_filters: [
{
clause: 'WHERE',
expressionType: 'SIMPLE',
subject: 'category',
operator: '==',
comparator: 'Electronics',
},
],
};
const queryObject = buildQueryObject(formDataWithFilters);
expect(queryObject.filters).toContainEqual({
col: 'category',
op: '==',
val: 'Electronics',
});
});
it('should handle SQL adhoc filters in extras', () => {
const formDataWithSQLFilters: QueryFormData = {
...baseFormData,
adhoc_filters: [
{
clause: 'WHERE',
expressionType: 'SQL',
sqlExpression: 'price > 100',
},
],
};
const queryObject = buildQueryObject(formDataWithSQLFilters);
expect(queryObject.extras?.where).toBe('(price > 100)');
});
it('should handle extra_form_data overrides', () => {
const formDataWithExtraFormData: QueryFormData = {
...baseFormData,
extra_form_data: {
time_range: 'Last week',
adhoc_filters: [
{
clause: 'WHERE',
expressionType: 'SIMPLE',
subject: 'status',
operator: '==',
comparator: 'active',
},
],
},
};
const queryObject = buildQueryObject(formDataWithExtraFormData);
expect(queryObject.time_range).toBe('Last week');
expect(queryObject.filters).toContainEqual({
col: 'status',
op: '==',
val: 'active',
});
});
it('should handle series_limit from limit field', () => {
const formDataWithLimit: QueryFormData = {
...baseFormData,
limit: 5,
};
const queryObject = buildQueryObject(formDataWithLimit);
expect(queryObject.series_limit).toBe(5);
});
it('should handle granularity', () => {
const formDataWithGranularity: QueryFormData = {
...baseFormData,
granularity: 'created_at',
};
const queryObject = buildQueryObject(formDataWithGranularity);
expect(queryObject.granularity).toBe('created_at');
});
});

View File

@@ -1,93 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import express from 'express';
import cors from 'cors';
import helmet from 'helmet';
import compression from 'compression';
import queryObjectRouter from './routes/queryObject';
const app = express();
const PORT = process.env.PORT || 3001;
const HOST = process.env.HOST || 'localhost';
// Security middleware
app.use(helmet());
// Enable CORS for Superset backend
app.use(cors({
origin: process.env.SUPERSET_ORIGINS?.split(',') || ['http://localhost:8088'],
credentials: true,
}));
// Compression middleware
app.use(compression());
// Body parsing middleware
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
// Request logging middleware
app.use((req, res, next) => {
const timestamp = new Date().toISOString();
console.log(`${timestamp} ${req.method} ${req.path}`);
next();
});
// Routes
app.use(queryObjectRouter);
// Global error handler
app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {
console.error('Unhandled error:', err);
res.status(500).json({
error: 'Internal server error',
message: process.env.NODE_ENV === 'development' ? err.message : undefined,
});
});
// 404 handler
app.use((req, res) => {
res.status(404).json({
error: 'Not found',
path: req.path,
});
});
// Start server
const server = app.listen(PORT, HOST, () => {
console.log(`🚀 Superset Query Sidecar Service running on http://${HOST}:${PORT}`);
console.log(`📊 Health check available at http://${HOST}:${PORT}/health`);
console.log(`🔧 Query object API available at http://${HOST}:${PORT}/api/v1/query-object`);
});
// Graceful shutdown
process.on('SIGTERM', () => {
console.log('SIGTERM received, shutting down gracefully');
server.close(() => {
console.log('Process terminated');
});
});
process.on('SIGINT', () => {
console.log('SIGINT received, shutting down gracefully');
server.close(() => {
console.log('Process terminated');
});
});
export default app;

View File

@@ -1,134 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
/* eslint-disable camelcase */
import {
QueryObject,
QueryFormData,
QueryFieldAliases,
isQueryFormMetric,
isDefined,
} from '../types';
import processFilters from '../utils/processFilters';
import extractExtras from '../utils/extractExtras';
import extractQueryFields from '../utils/extractQueryFields';
import { overrideExtraFormData } from '../utils/processExtraFormData';
/**
* Build the common segments of all query objects (e.g. the granularity field derived from
* SQLAlchemy). The segments specific to each viz type is constructed in the
* buildQuery method for each viz type (see `wordcloud/buildQuery.ts` for an example).
* Note the type of the formData argument passed in here is the type of the formData for a
* specific viz, which is a subtype of the generic formData shared among all viz types.
*/
export default function buildQueryObject<T extends QueryFormData>(
formData: T,
queryFields?: QueryFieldAliases,
): QueryObject {
const {
annotation_layers = [],
extra_form_data,
time_range,
since,
until,
row_limit,
row_offset,
order_desc,
limit,
timeseries_limit_metric,
granularity,
url_params = {},
custom_params = {},
series_columns,
series_limit,
series_limit_metric,
group_others_when_limit_reached,
...residualFormData
} = formData;
const {
adhoc_filters: appendAdhocFilters = [],
filters: appendFilters = [],
custom_form_data = {},
...overrides
} = extra_form_data || {};
const numericRowLimit = Number(row_limit);
const numericRowOffset = Number(row_offset);
const { metrics, columns, orderby } = extractQueryFields(
residualFormData,
queryFields,
);
// collect all filters for conversion to simple filters/freeform clauses
const extras = extractExtras(formData);
const { filters: extraFilters } = extras;
const filterFormData = {
...formData,
...extras,
filters: [...extraFilters, ...appendFilters],
adhoc_filters: [...(formData.adhoc_filters || []), ...appendAdhocFilters],
};
const extrasAndfilters = processFilters(filterFormData);
const normalizeSeriesLimitMetric = (metric: any) => {
if (isQueryFormMetric(metric)) {
return metric;
}
return undefined;
};
let queryObject: QueryObject = {
// fallback `null` to `undefined` so they won't be sent to the backend
// (JSON.stringify will ignore `undefined`.)
time_range: time_range || undefined,
since: since || undefined,
until: until || undefined,
granularity: granularity || undefined,
...extras,
...extrasAndfilters,
columns,
metrics,
orderby,
annotation_layers,
row_limit:
row_limit == null || Number.isNaN(numericRowLimit)
? undefined
: numericRowLimit,
row_offset:
row_offset == null || Number.isNaN(numericRowOffset)
? undefined
: numericRowOffset,
series_columns,
series_limit: series_limit ?? (isDefined(limit) ? Number(limit) : 0),
series_limit_metric:
normalizeSeriesLimitMetric(series_limit_metric) ??
timeseries_limit_metric ??
undefined,
group_others_when_limit_reached: group_others_when_limit_reached ?? false,
order_desc: typeof order_desc === 'undefined' ? true : order_desc,
url_params: url_params || undefined,
custom_params,
};
// override extra form data used by native and cross filters
queryObject = overrideExtraFormData(queryObject, overrides);
return { ...queryObject, custom_form_data };
}

View File

@@ -1,91 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import { Request, Response, Router } from 'express';
import buildQueryObject from '../query/buildQueryObject';
import { QueryFormData, QueryFieldAliases } from '../types';
const router = Router();
interface BuildQueryObjectRequest {
form_data: QueryFormData;
query_fields?: QueryFieldAliases;
}
interface BuildQueryObjectResponse {
query_object: any;
error?: string;
}
/**
* POST /api/v1/query-object
*
* Build a QueryObject from form_data
*
* Request body:
* - form_data: The form data from Superset frontend
* - query_fields: Optional query field aliases for visualization-specific mappings
*
* Response:
* - query_object: The computed QueryObject
* - error: Error message if processing failed
*/
router.post('/api/v1/query-object', (req: Request, res: Response) => {
try {
const { form_data, query_fields }: BuildQueryObjectRequest = req.body;
if (!form_data) {
return res.status(400).json({
error: 'form_data is required',
} as BuildQueryObjectResponse);
}
// Validate required form_data fields
if (!form_data.datasource || !form_data.viz_type) {
return res.status(400).json({
error: 'form_data must include datasource and viz_type',
} as BuildQueryObjectResponse);
}
const queryObject = buildQueryObject(form_data, query_fields);
res.json({
query_object: queryObject,
} as BuildQueryObjectResponse);
} catch (error: any) {
console.error('Error building query object:', error);
res.status(500).json({
error: `Failed to build query object: ${error.message}`,
} as BuildQueryObjectResponse);
}
});
/**
* GET /health
*
* Health check endpoint
*/
router.get('/health', (req: Request, res: Response) => {
res.json({
status: 'healthy',
timestamp: new Date().toISOString(),
version: process.env.npm_package_version || '1.0.0',
});
});
export default router;

View File

@@ -1,213 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
/* eslint-disable camelcase */
// Basic types for form data and query objects
export interface JsonObject {
[key: string]: any;
}
// Metric types
export interface SavedMetric {
metric_name: string;
expression?: string;
label?: string;
}
export interface AdhocMetric {
aggregate: string;
column?: any;
expressionType: 'SIMPLE' | 'SQL';
hasCustomLabel?: boolean;
label: string;
sqlExpression?: string;
optionName?: string;
}
export type QueryFormMetric = SavedMetric | AdhocMetric | string;
// Column types
export interface PhysicalColumn {
column_name: string;
type?: string;
}
export interface AdhocColumn {
hasCustomLabel?: boolean;
label: string;
sqlExpression: string;
expressionType: 'SQL';
optionName?: string;
}
export type QueryFormColumn = PhysicalColumn | AdhocColumn | string;
// Filter types
export type BinaryOperator =
| '==' | '!=' | '>' | '<' | '>=' | '<='
| 'LIKE' | 'ILIKE' | 'REGEX' | 'NOT REGEX';
export type SetOperator = 'IN' | 'NOT IN';
export interface AdhocFilter {
clause: 'WHERE' | 'HAVING';
comparator?: any;
expressionType: 'SIMPLE' | 'SQL';
operator?: BinaryOperator | SetOperator;
subject?: string | AdhocColumn;
sqlExpression?: string;
filterOptionName?: string;
}
export interface QueryObjectFilterClause {
col: string;
op: BinaryOperator | SetOperator;
val: any;
}
// Order by types
export type QueryFormOrderBy = [QueryFormColumn | QueryFormMetric | {}, boolean] | [];
// Annotation types
export interface AnnotationLayer {
annotationType: string;
name: string;
show: boolean;
sourceType?: string;
value?: string;
[key: string]: any;
}
// Time range types
export interface TimeRange {
time_range?: string;
since?: string;
until?: string;
}
// Extra form data types
export interface ExtraFormDataAppend {
adhoc_filters?: AdhocFilter[];
filters?: QueryObjectFilterClause[];
interactive_drilldown?: string[];
interactive_groupby?: string[];
interactive_highlight?: string[];
custom_form_data?: JsonObject;
}
export interface ExtraFormDataOverride {
granularity_sqla?: string;
granularity?: string;
time_range?: string;
time_column?: string;
time_grain?: string;
time_compare?: string[];
relative_start?: string;
relative_end?: string;
time_grain_sqla?: string;
}
export type ExtraFormData = ExtraFormDataAppend & ExtraFormDataOverride;
// Query extras interface
export interface QueryObjectExtras {
having?: string;
where?: string;
time_grain_sqla?: string;
time_range_endpoints?: [string, string];
relative_start?: string;
relative_end?: string;
time_compare?: string[];
[key: string]: any;
}
// Main form data interface
export interface BaseFormData extends TimeRange {
datasource: string;
viz_type: string;
metrics?: QueryFormMetric[];
where?: string;
columns?: QueryFormColumn[];
groupby?: QueryFormColumn[];
all_columns?: QueryFormColumn[];
adhoc_filters?: AdhocFilter[] | null;
extra_form_data?: ExtraFormData;
order_desc?: boolean;
limit?: number;
row_limit?: string | number | null;
row_offset?: string | number | null;
series_columns?: QueryFormColumn[];
series_limit?: number;
series_limit_metric?: QueryFormMetric;
annotation_layers?: AnnotationLayer[];
url_params?: Record<string, string>;
custom_params?: Record<string, string>;
[key: string]: any;
}
export interface SqlaFormData extends BaseFormData {
granularity?: string;
granularity_sqla?: string;
time_grain_sqla?: string;
having?: string;
}
export type QueryFormData = SqlaFormData;
// Query object interface
export interface QueryObject {
time_range?: string;
since?: string;
until?: string;
granularity?: string;
columns?: QueryFormColumn[];
metrics?: QueryFormMetric[];
orderby?: QueryFormOrderBy[];
annotation_layers?: AnnotationLayer[];
row_limit?: number;
row_offset?: number;
series_columns?: QueryFormColumn[];
series_limit?: number;
series_limit_metric?: QueryFormMetric;
group_others_when_limit_reached?: boolean;
order_desc?: boolean;
url_params?: Record<string, string>;
custom_params?: Record<string, string>;
extras?: QueryObjectExtras;
filters?: QueryObjectFilterClause[];
custom_form_data?: JsonObject;
}
// Query field aliases
export type QueryFieldAliases = {
[key: string]: 'metrics' | 'columns' | 'groupby';
};
// Utility functions
export function isAdhocMetric(metric: any): metric is AdhocMetric {
return metric && typeof metric === 'object' && 'expressionType' in metric;
}
export function isQueryFormMetric(metric: any): metric is QueryFormMetric {
return typeof metric === 'string' || isAdhocMetric(metric) ||
(metric && typeof metric === 'object' && 'metric_name' in metric);
}
export function isDefined<T>(value: T | undefined | null): value is T {
return value !== undefined && value !== null;
}

View File

@@ -1,78 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import {
QueryFormData,
QueryObjectExtras,
QueryObjectFilterClause,
} from '../types';
interface ExtrasResult extends QueryObjectExtras {
filters: QueryObjectFilterClause[];
}
/**
* Extract extras and filters from form data
*/
export default function extractExtras(formData: QueryFormData): ExtrasResult {
const {
where,
having,
time_grain_sqla,
granularity_sqla,
granularity,
extra_filters = [],
} = formData;
const extras: QueryObjectExtras = {};
const filters: QueryObjectFilterClause[] = [];
// Add SQL clauses to extras
if (where) {
extras.where = where;
}
if (having) {
extras.having = having;
}
if (time_grain_sqla) {
extras.time_grain_sqla = time_grain_sqla;
}
// Handle granularity - prefer granularity_sqla over granularity
const timeColumn = granularity_sqla || granularity;
if (timeColumn) {
// Time column handling would go here if needed
}
// Convert extra_filters to QueryObjectFilterClause format
if (extra_filters && Array.isArray(extra_filters)) {
for (const filter of extra_filters) {
if (filter.col && filter.op && filter.val !== undefined) {
filters.push({
col: filter.col,
op: filter.op,
val: filter.val,
});
}
}
}
return {
...extras,
filters,
};
}

View File

@@ -1,73 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import {
QueryFormData,
QueryFormMetric,
QueryFormColumn,
QueryFormOrderBy,
QueryFieldAliases,
} from '../types';
interface QueryFieldsResult {
metrics?: QueryFormMetric[];
columns?: QueryFormColumn[];
orderby?: QueryFormOrderBy[];
}
/**
* Extract query fields (metrics, columns, orderby) from form data
*/
export default function extractQueryFields(
formData: QueryFormData,
queryFieldAliases?: QueryFieldAliases,
): QueryFieldsResult {
const result: QueryFieldsResult = {};
// Extract metrics
if (formData.metrics && formData.metrics.length > 0) {
result.metrics = formData.metrics;
}
// Extract columns - prefer 'columns' over 'groupby'
const columns = formData.columns || formData.groupby;
if (columns && columns.length > 0) {
result.columns = columns;
}
// Handle query field aliases if provided
if (queryFieldAliases) {
for (const [formFieldName, queryField] of Object.entries(queryFieldAliases)) {
const formValue = (formData as any)[formFieldName];
if (formValue && Array.isArray(formValue) && formValue.length > 0) {
if (queryField === 'metrics') {
result.metrics = formValue;
} else if (queryField === 'columns' || queryField === 'groupby') {
result.columns = formValue;
}
}
}
}
// Extract orderby - this can be complex as it depends on the form structure
// For now, we'll handle basic cases
if (formData.orderby && Array.isArray(formData.orderby)) {
result.orderby = formData.orderby as QueryFormOrderBy[];
}
return result;
}

View File

@@ -1,57 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import { QueryObject, ExtraFormDataOverride } from '../types';
/**
* Override extra form data used by native and cross filters
*/
export function overrideExtraFormData(
queryObject: QueryObject,
overrides: ExtraFormDataOverride,
): QueryObject {
const result = { ...queryObject };
// Override top-level properties
if (overrides.time_range !== undefined) {
result.time_range = overrides.time_range;
}
if (overrides.granularity !== undefined) {
result.granularity = overrides.granularity;
}
if (overrides.granularity_sqla !== undefined) {
result.granularity = overrides.granularity_sqla;
}
// Override extras properties
if (result.extras) {
if (overrides.relative_start !== undefined) {
result.extras.relative_start = overrides.relative_start;
}
if (overrides.relative_end !== undefined) {
result.extras.relative_end = overrides.relative_end;
}
if (overrides.time_grain_sqla !== undefined) {
result.extras.time_grain_sqla = overrides.time_grain_sqla;
}
if (overrides.time_compare !== undefined) {
result.extras.time_compare = overrides.time_compare;
}
}
return result;
}

View File

@@ -1,72 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import {
AdhocFilter,
QueryObjectFilterClause,
QueryFormData,
QueryObjectExtras,
} from '../types';
interface FilterProcessorInput extends QueryFormData {
extras: QueryObjectExtras;
filters: QueryObjectFilterClause[];
adhoc_filters: AdhocFilter[];
}
interface FilterProcessorResult {
extras: QueryObjectExtras;
filters: QueryObjectFilterClause[];
}
/**
* Process filters from form data into QueryObject format
*/
export default function processFilters(
formData: FilterProcessorInput,
): FilterProcessorResult {
const { filters = [], adhoc_filters = [], extras = {} } = formData;
// Convert adhoc filters to simple filters where possible
const processedFilters: QueryObjectFilterClause[] = [...filters];
const processedExtras: QueryObjectExtras = { ...extras };
// Process adhoc filters
for (const adhocFilter of adhoc_filters) {
if (adhocFilter.expressionType === 'SIMPLE' && adhocFilter.subject && adhocFilter.operator) {
// Convert simple adhoc filters to QueryObjectFilterClause
if (typeof adhocFilter.subject === 'string') {
processedFilters.push({
col: adhocFilter.subject,
op: adhocFilter.operator as any,
val: adhocFilter.comparator,
});
}
} else if (adhocFilter.expressionType === 'SQL' && adhocFilter.sqlExpression) {
// Add SQL filters to WHERE clause in extras
const clause = adhocFilter.clause === 'HAVING' ? 'having' : 'where';
const existingClause = processedExtras[clause] || '';
const separator = existingClause ? ' AND ' : '';
processedExtras[clause] = existingClause + separator + `(${adhocFilter.sqlExpression})`;
}
}
return {
extras: processedExtras,
filters: processedFilters,
};
}

View File

@@ -1,34 +0,0 @@
{
"compilerOptions": {
"target": "ES2020",
"lib": ["ES2020"],
"module": "commonjs",
"declaration": true,
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"moduleResolution": "node",
"baseUrl": "./",
"paths": {
"@/*": ["src/*"]
},
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": [
"src/**/*"
],
"exclude": [
"node_modules",
"dist",
"**/*.test.ts"
]
}

View File

@@ -17,6 +17,6 @@ specific language governing permissions and limitations
under the License.
-->
## Change Log
# Apache Superset SDK
Changelogs will be added once we have the first stable release.
This is an SDK tool used for bundling Apache Superset extensions.

View File

@@ -16,35 +16,20 @@
# under the License.
[project]
name = "apache-superset-extensions-cli"
version = "0.0.1rc2"
description = "Official command-line interface for building, bundling, and managing Apache Superset extensions"
readme = "README.md"
name = "apache-superset-cli"
version = "0.0.1"
description = "SDK to build Apache Superset extensions"
authors = [
{ name = "Apache Software Foundation", email = "dev@superset.apache.org" },
]
license = { file="LICENSE.txt" }
requires-python = ">=3.10"
keywords = ["superset", "apache", "cli", "extensions", "analytics", "business-intelligence", "development-tools"]
classifiers = [
"Development Status :: 3 - Alpha",
"Environment :: Console",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Database",
"Topic :: Scientific/Engineering :: Visualization",
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Software Distribution",
]
dependencies = [
# no bounds for apache-superset-core until we have a stable version
"apache-superset-core",
"apache-superset-core>=0.0.1, <0.2",
"click>=8.0.3",
"jinja2>=3.1.6",
"semver>=3.0.4",
@@ -52,13 +37,6 @@ dependencies = [
"watchdog>=6.0.0",
]
[project.urls]
Homepage = "https://superset.apache.org/"
Documentation = "https://superset.apache.org/docs/"
Repository = "https://github.com/apache/superset"
"Bug Tracker" = "https://github.com/apache/superset/issues"
Changelog = "https://github.com/apache/superset/blob/master/CHANGELOG.md"
[project.optional-dependencies]
test = [
"pytest",
@@ -71,17 +49,11 @@ requires = ["setuptools>=76.0.0", "wheel"]
build-backend = "setuptools.build_meta"
[tool.setuptools]
packages = ["superset_cli"]
package-dir = { "" = "src" }
include-package-data = true
[tool.setuptools.packages.find]
where = ["src"]
[tool.setuptools.package-data]
superset_extensions_cli = ["templates/**/*"]
[project.scripts]
superset-extensions = "superset_extensions_cli.cli:app"
superset-extensions = "superset_cli.cli:app"
[tool.pytest.ini_options]
testpaths = ["tests"]
@@ -92,7 +64,7 @@ addopts = [
"--strict-markers",
"--strict-config",
"--verbose",
"--cov=superset_extensions_cli",
"--cov=superset_cli",
"--cov-report=term-missing",
"--cov-report=html:htmlcov"
]
@@ -104,7 +76,7 @@ markers = [
]
[tool.coverage.run]
source = ["src/superset_extensions_cli"]
source = ["src/superset_cli"]
omit = ["*/tests/*", "*/test_*"]
[tool.coverage.report]
@@ -122,4 +94,4 @@ exclude_lines = [
]
[tool.ruff.lint.per-file-ignores]
"src/superset_extensions_cli/*" = ["TID251"]
"src/superset_cli/*" = ["TID251"]

View File

@@ -32,8 +32,8 @@ from superset_core.extensions.types import Manifest, Metadata
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
from superset_extensions_cli.constants import MIN_NPM_VERSION
from superset_extensions_cli.utils import read_json, read_toml
from superset_cli.constants import MIN_NPM_VERSION
from superset_cli.utils import read_json, read_toml
REMOTE_ENTRY_REGEX = re.compile(r"^remoteEntry\..+\.js$")
FRONTEND_DIST_REGEX = re.compile(r"/frontend/dist")

Some files were not shown because too many files have changed in this diff Show More