Compare commits

..

1 Commits

Author SHA1 Message Date
Evan Rusackas
b7c4b9d655 chore(superset-ui): publish v0.21.0 2024-12-10 16:57:42 -07:00
1835 changed files with 112866 additions and 96181 deletions

View File

@@ -18,7 +18,6 @@
# https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories
---
github:
del_branch_on_merge: true
description: "Apache Superset is a Data Visualization and Data Exploration Platform"
homepage: https://superset.apache.org/
labels:
@@ -73,16 +72,18 @@ github:
- cypress-matrix (3, chrome)
- cypress-matrix (4, chrome)
- cypress-matrix (5, chrome)
- dependency-review
- frontend-build
- pre-commit (current)
- pre-commit (next)
- pre-commit (previous)
- test-mysql
- test-postgres (current)
- test-postgres (next)
- test-postgres-hive
- test-postgres-presto
- test-sqlite
- unit-tests (current)
- unit-tests (next)
required_pull_request_reviews:
dismiss_stale_reviews: false

View File

@@ -34,6 +34,7 @@
**/*.sqllite
**/*.swp
**/.terser-plugin-cache/
**/.storybook/
**/node_modules/
tests/
@@ -41,8 +42,6 @@ docs/
install/
superset-frontend/cypress-base/
superset-frontend/coverage/
superset-frontend/.temp_cache/
superset/static/assets/
superset-websocket/dist/
venv
.venv

10
.github/CODEOWNERS vendored
View File

@@ -12,21 +12,21 @@
# Notify Helm Chart maintainers about changes in it
/helm/superset/ @craig-rueda @dpgaspar @villebro @nytai @michael-s-molina @mistercrunch @rusackas @Antonio-RiveroMartnez
/helm/superset/ @craig-rueda @dpgaspar @villebro @nytai @michael-s-molina
# Notify E2E test maintainers of changes
/superset-frontend/cypress-base/ @sadpandajoe @geido @eschutho @rusackas @betodealmeida @mistercrunch
/superset-frontend/cypress-base/ @jinghua-qa @geido @eschutho @rusackas @betodealmeida
# Notify PMC members of changes to GitHub Actions
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @kgabryje @dpgaspar
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @john-bodley @kgabryje @dpgaspar
# Notify PMC members of changes to required GitHub Actions
/.asf.yaml @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @kgabryje @dpgaspar @Antonio-RiveroMartnez
/.asf.yaml @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @john-bodley @kgabryje @dpgaspar
# Maps are a finicky contribution process we care about
# Maps are a finnicky contribution process we care about
**/*.geojson @villebro @rusackas
/superset-frontend/plugins/legacy-plugin-chart-country-map/ @villebro @rusackas

View File

@@ -1,23 +0,0 @@
name: Label Draft PRs
on:
pull_request:
types:
- opened
- converted_to_draft
jobs:
label-draft:
runs-on: ubuntu-latest
steps:
- name: Check if the PR is a draft
id: check-draft
uses: actions/github-script@v6
with:
script: |
const isDraft = context.payload.pull_request.draft;
core.setOutput('isDraft', isDraft);
- name: Add `review:draft` Label
if: steps.check-draft.outputs.isDraft == 'true'
uses: actions-ecosystem/action-add-labels@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
labels: "review:draft"

View File

@@ -26,12 +26,11 @@ runs:
shell: bash
run: |
if [ "${{ inputs.python-version }}" = "current" ]; then
echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
elif [ "${{ inputs.python-version }}" = "next" ]; then
# currently disabled in GHA matrixes because of library compatibility issues
echo "PYTHON_VERSION=3.12" >> $GITHUB_ENV
elif [ "${{ inputs.python-version }}" = "previous" ]; then
echo "PYTHON_VERSION=3.10" >> $GITHUB_ENV
elif [ "${{ inputs.python-version }}" = "next" ]; then
echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
elif [ "${{ inputs.python-version }}" = "previous" ]; then
echo "PYTHON_VERSION=3.9" >> $GITHUB_ENV
else
echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
fi
@@ -44,15 +43,11 @@ runs:
run: |
if [ "${{ inputs.install-superset }}" = "true" ]; then
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
pip install --upgrade pip setuptools wheel uv
if [ "${{ inputs.requirements-type }}" = "dev" ]; then
uv pip install --system -r requirements/development.txt
elif [ "${{ inputs.requirements-type }}" = "base" ]; then
uv pip install --system -r requirements/base.txt
fi
uv pip install --system -e .
fi
shell: bash

View File

@@ -1,69 +0,0 @@
name: "Setup Docker Environment"
description: "Reusable steps for setting up QEMU, Docker Buildx, DockerHub login, Supersetbot, and optionally Docker Compose"
inputs:
build:
description: "Used for building?"
required: false
default: "false"
dockerhub-user:
description: "DockerHub username"
required: false
dockerhub-token:
description: "DockerHub token"
required: false
install-docker-compose:
description: "Flag to install Docker Compose"
required: false
default: "true"
login-to-dockerhub:
description: "Whether you want to log into dockerhub"
required: false
default: "true"
outputs: {}
runs:
using: "composite"
steps:
- name: Set up QEMU
if: ${{ inputs.build == 'true' }}
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
if: ${{ inputs.build == 'true' }}
uses: docker/setup-buildx-action@v3
- name: Try to login to DockerHub
if: ${{ inputs.login-to-dockerhub == 'true' }}
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ inputs.dockerhub-user }}
password: ${{ inputs.dockerhub-token }}
- name: Install Docker Compose
if: ${{ inputs.install-docker-compose == 'true' }}
shell: bash
run: |
sudo apt-get update
sudo apt-get install -y ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
# Download and save the Docker GPG key in the correct format
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
# Ensure the key file is readable
sudo chmod a+r /etc/apt/keyrings/docker.gpg
# Add the Docker repository using the correct key
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# Update package lists and install Docker Compose plugin
sudo apt update
sudo apt install -y docker-compose-plugin
- name: Docker Version Info
shell: bash
run: docker info

View File

@@ -1,5 +1,4 @@
version: 2
enable-beta-ecosystems: true
updates:
- package-ecosystem: "github-actions"
@@ -22,14 +21,10 @@ updates:
versioning-strategy: increase
# NOTE: `uv` support is in beta, more details here:
# https://github.com/dependabot/dependabot-core/pull/10040#issuecomment-2696978430
- package-ecosystem: "uv"
directory: "requirements/"
open-pull-requests-limit: 10
labels:
- uv
- dependabot
# - package-ecosystem: "pip"
# NOTE: as dependabot isn't compatible with our python
# dependency setup (pip-compile-multi), we'll be using
# `supersetbot` instead
- package-ecosystem: "npm"
directory: ".github/actions"
@@ -328,10 +323,6 @@ updates:
- package-ecosystem: "npm"
directory: "/superset-frontend/packages/superset-ui-core/"
ignore:
# not until React >= 18.0.0
- dependency-name: "react-markdown"
- dependency-name: "remark-gfm"
schedule:
interval: "monthly"
labels:

View File

@@ -23,7 +23,7 @@ on:
jobs:
bump-python-package:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
actions: write
contents: write
@@ -45,8 +45,8 @@ jobs:
with:
python-version: "3.10"
- name: Install uv
run: pip install uv
- name: Install pip-compile-multi
run: pip install pip-compile-multi
- name: supersetbot bump-python -p "${{ github.event.inputs.package }}"
env:

View File

@@ -9,7 +9,7 @@ on:
jobs:
cancel-duplicate-runs:
name: Cancel duplicate workflow runs
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
actions: write
contents: read

View File

@@ -1,44 +0,0 @@
name: Check python dependencies
on:
push:
branches:
- "master"
- "[0-9].[0-9]*"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
# cancel previous workflow jobs for PRs
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
check-python-deps:
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
depth: 1
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
- name: Run uv
if: steps.check.outputs.python
run: ./scripts/uv-pip-compile.sh
- name: Check for uncommitted changes
run: |
if [[ -n "$(git diff)" ]]; then
echo "ERROR: The pinned dependencies are not up-to-date."
echo "Please run './scripts/uv-pip-compile.sh' and commit the changes."
exit 1
else
echo "Pinned dependencies are up-to-date."
fi

View File

@@ -19,7 +19,7 @@ concurrency:
jobs:
check_db_migration_conflict:
name: Check DB migration conflict
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write

View File

@@ -17,7 +17,7 @@ concurrency:
jobs:
analyze:
name: Analyze
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
actions: read
contents: read

View File

@@ -5,32 +5,19 @@
# Source repository: https://github.com/actions/dependency-review-action
# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
name: "Dependency Review"
on:
push:
branches:
- "master"
- "[0-9].[0-9]*"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
# cancel previous workflow jobs for PRs
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
- name: "Dependency Review"
uses: actions/dependency-review-action@v4
continue-on-error: true
with:
fail-on-severity: critical
# compatible/incompatible licenses addressed here: https://www.apache.org/legal/resolved.html
@@ -46,24 +33,3 @@ jobs:
# pkg:npm/node-forge@1.3.1
# selecting BSD-3-Clause licensing terms for node-forge to ensure compatibility with Apache
allow-dependencies-licenses: pkg:npm/store2@2.14.2, pkg:npm/applitools/core, pkg:npm/applitools/core-base, pkg:npm/applitools/css-tree, pkg:npm/applitools/ec-client, pkg:npm/applitools/eg-socks5-proxy-server, pkg:npm/applitools/eyes, pkg:npm/applitools/eyes-cypress, pkg:npm/applitools/nml-client, pkg:npm/applitools/tunnel-client, pkg:npm/applitools/utils, pkg:npm/node-forge@1.3.1, pkg:npm/rgbcolor, pkg:npm/jszip@3.10.1
python-dependency-liccheck:
runs-on: ubuntu-22.04
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
- name: Setup Python
uses: ./.github/actions/setup-backend/
with:
requirements-type: base
- name: "Set up liccheck"
run: |
uv pip install --system liccheck
- name: "Run liccheck"
run: |
# run the checks
liccheck -R output.txt
# Print the report
cat output.txt

View File

@@ -14,22 +14,21 @@ concurrency:
cancel-in-progress: true
jobs:
setup_matrix:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
outputs:
matrix_config: ${{ steps.set_matrix.outputs.matrix_config }}
steps:
- id: set_matrix
run: |
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev", "lean"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize", "py311"]'; fi)
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize", "py311"]'; fi)
echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
echo $GITHUB_OUTPUT
docker-build:
name: docker-build
needs: setup_matrix
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
build_preset: ${{fromJson(needs.setup_matrix.outputs.matrix_config)}}
@@ -37,7 +36,6 @@ jobs:
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
IMAGE_TAG: apache/superset:GHA-${{ matrix.build_preset }}-${{ github.run_id }}
steps:
@@ -52,13 +50,21 @@ jobs:
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Docker Environment
- name: Set up QEMU
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: ./.github/actions/setup-docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: docker/setup-buildx-action@v3
- name: Try to login to DockerHub
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
continue-on-error: true
uses: docker/login-action@v3
with:
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
build: "true"
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup supersetbot
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
@@ -73,65 +79,12 @@ jobs:
# Single platform builds in pull_request context to speed things up
if [ "${{ github.event_name }}" = "push" ]; then
PLATFORM_ARG="--platform linux/arm64 --platform linux/amd64"
# can only --load images in single-platform builds
PUSH_OR_LOAD="--push"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
PLATFORM_ARG="--platform linux/amd64"
PUSH_OR_LOAD="--load"
fi
supersetbot docker \
$PUSH_OR_LOAD \
--preset ${{ matrix.build_preset }} \
--context "$EVENT" \
--context-ref "$RELEASE" $FORCE_LATEST \
--extra-flags "--build-arg INCLUDE_CHROMIUM=false --tag $IMAGE_TAG" \
$PLATFORM_ARG
# in the context of push (using multi-platform build), we need to pull the image locally
- name: Docker pull
if: github.event_name == 'push' && (steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker)
run: docker pull $IMAGE_TAG
- name: Print docker stats
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
run: |
echo "SHA: ${{ github.sha }}"
echo "IMAGE: $IMAGE_TAG"
docker images $IMAGE_TAG
docker history $IMAGE_TAG
- name: docker-compose sanity check
if: (steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker) && (matrix.build_preset == 'dev' || matrix.build_preset == 'lean')
shell: bash
run: |
export SUPERSET_BUILD_TARGET=${{ matrix.build_preset }}
# This should reuse the CACHED image built in the previous steps
docker compose build superset-init --build-arg DEV_MODE=false --build-arg INCLUDE_CHROMIUM=false
docker compose up superset-init --exit-code-from superset-init
docker-compose-image-tag:
runs-on: ubuntu-24.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Docker Environment
if: steps.check.outputs.docker
uses: ./.github/actions/setup-docker
with:
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
build: "false"
install-docker-compose: "true"
- name: docker-compose sanity check
if: steps.check.outputs.docker
shell: bash
run: |
docker compose -f docker-compose-image-tag.yml up superset-init --exit-code-from superset-init

View File

@@ -8,7 +8,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -23,7 +23,7 @@ jobs:
build:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
defaults:
run:
working-directory: superset-embedded-sdk
@@ -31,7 +31,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version-file: './superset-embedded-sdk/.nvmrc'
node-version: "20"
registry-url: 'https://registry.npmjs.org'
- run: npm ci
- run: npm run ci:release

View File

@@ -13,7 +13,7 @@ concurrency:
jobs:
embedded-sdk-test:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
defaults:
run:
working-directory: superset-embedded-sdk
@@ -21,7 +21,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version-file: './superset-embedded-sdk/.nvmrc'
node-version: "20"
registry-url: 'https://registry.npmjs.org'
- run: npm ci
- run: npm test

View File

@@ -6,7 +6,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -22,7 +22,7 @@ jobs:
needs: config
if: needs.config.outputs.has-secrets
name: Cleanup ephemeral envs
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
pull-requests: write
steps:

View File

@@ -1,166 +1,149 @@
name: Ephemeral env workflow
# Example manual trigger:
# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field label_name="testenv-up" --field issue_number=666
# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
on:
pull_request_target:
types:
- labeled
issue_comment:
types: [created]
workflow_dispatch:
inputs:
label_name:
description: 'Label name to simulate label-based /testenv trigger'
comment_body:
description: 'Comment body to simulate /testenv command'
required: true
default: 'testenv-up'
default: '/testenv up'
issue_number:
description: 'Issue or PR number'
required: true
jobs:
ephemeral-env-label:
ephemeral-env-comment:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-label
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
cancel-in-progress: true
name: Evaluate ephemeral env label trigger
runs-on: ubuntu-24.04
name: Evaluate ephemeral env comment trigger (/testenv)
runs-on: ubuntu-22.04
permissions:
pull-requests: write
outputs:
slash-command: ${{ steps.eval-label.outputs.result }}
slash-command: ${{ steps.eval-body.outputs.result }}
feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
sha: ${{ steps.get-sha.outputs.sha }}
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
steps:
- name: Check for the "testenv-up" label
id: eval-label
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
LABEL_NAME="${{ github.event.inputs.label_name }}"
else
LABEL_NAME="${{ github.event.label.name }}"
fi
- name: Debug
run: |
echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}"
echo "Evaluating label: $LABEL_NAME"
- name: Eval comment body for /testenv slash command
uses: actions/github-script@v7
env:
COMMENT_BODY: ${{ github.event.inputs.comment_body || github.event.comment.body }}
id: eval-body
with:
result-encoding: string
script: |
const pattern = /^\/testenv (up|down)/;
const result = pattern.exec(process.env.COMMENT_BODY || '');
return result === null ? 'noop' : result[1];
if [[ "$LABEL_NAME" == "testenv-up" ]]; then
echo "result=up" >> $GITHUB_OUTPUT
else
echo "result=noop" >> $GITHUB_OUTPUT
fi
- name: Get event SHA
id: get-sha
if: steps.eval-label.outputs.result == 'up'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
let prSha;
// If event is workflow_dispatch, use the issue_number from inputs
if (context.eventName === "workflow_dispatch") {
const prNumber = "${{ github.event.inputs.issue_number }}";
if (!prNumber) {
console.log("No PR number found.");
return;
}
// Fetch PR details using the provided issue_number
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: prNumber
});
prSha = pr.head.sha;
} else {
// If it's not workflow_dispatch, use the PR head sha from the event
prSha = context.payload.pull_request.head.sha;
}
console.log(`PR SHA: ${prSha}`);
core.setOutput("sha", prSha);
- name: Looking for feature flags in PR description
uses: actions/github-script@v7
id: eval-feature-flags
if: steps.eval-label.outputs.result == 'up'
with:
script: |
const description = context.payload.pull_request
? context.payload.pull_request.body || ''
: context.payload.inputs.pr_description || '';
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...description.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
};
results.push(config);
});
return results;
- name: Reply with confirmation comment
uses: actions/github-script@v7
if: steps.eval-label.outputs.result == 'up'
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const action = '${{ steps.eval-label.outputs.result }}';
const user = context.actor;
const runId = context.runId;
const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
const issueNumber = context.payload.pull_request
? context.payload.pull_request.number
: context.payload.inputs.issue_number;
if (!issueNumber) {
throw new Error("Issue number is not available.");
}
const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}).` +
` Action: **${action}**.` +
` More information on [how to use or configure ephemeral environments]` +
`(https://superset.apache.org/docs/contributing/howtos/#github-ephemeral-environments)`;
- name: Looking for feature flags
uses: actions/github-script@v7
env:
COMMENT_BODY: ${{ github.event.inputs.comment_body || github.event.comment.body }}
id: eval-feature-flags
with:
script: |
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...process.env.COMMENT_BODY.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
};
results.push(config);
});
return results;
- name: Limit to committers
if: >
steps.eval-body.outputs.result != 'noop' &&
github.event_name == 'issue_comment' &&
github.event.comment.author_association != 'MEMBER' &&
github.event.comment.author_association != 'OWNER'
uses: actions/github-script@v7
with:
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
});
core.setFailed(errMsg);
- name: Reply with confirmation comment
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const issueNumber = ${{ github.event.inputs.issue_number || github.event.issue.number }};
const user = '${{ github.event.comment.user.login || github.actor }}';
const action = '${{ steps.eval-body.outputs.result }}';
const runId = context.runId;
const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}).`;
if (action !== 'noop') {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
body,
});
}
else {
core.setFailed('No ephemeral environment action detected.');
}
ephemeral-docker-build:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-build
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
cancel-in-progress: true
needs: ephemeral-env-label
if: needs.ephemeral-env-label.outputs.slash-command == 'up'
needs: ephemeral-env-comment
if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
name: ephemeral-docker-build
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )"
- name: Get Info from comment
uses: actions/github-script@v7
id: get-pr-info
with:
script: |
const request = {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
};
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
const pr = await github.rest.pulls.get(request);
return pr.data;
- name: Debug
id: get-sha
run: |
echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )"
uses: actions/checkout@v4
with:
ref: ${{ needs.ephemeral-env-label.outputs.sha }}
ref: ${{ steps.get-sha.outputs.sha }}
persist-credentials: false
- name: Setup Docker Environment
uses: ./.github/actions/setup-docker
with:
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
build: "true"
install-docker-compose: "false"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
@@ -170,12 +153,9 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
supersetbot docker \
--push \
--load \
--preset ci \
--platform linux/amd64 \
--context-ref "$RELEASE" \
--extra-flags "--build-arg INCLUDE_CHROMIUM=false"
--context-ref "$RELEASE"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
@@ -193,141 +173,135 @@ jobs:
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: superset-ci
IMAGE_TAG: apache/superset:${{ needs.ephemeral-env-label.outputs.sha }}-ci
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-$PR_NUMBER-ci
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
needs: [ephemeral-env-label, ephemeral-docker-build]
if: needs.ephemeral-env-label.outputs.slash-command == 'up'
needs: [ephemeral-env-comment, ephemeral-docker-build]
if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
name: Spin up an ephemeral environment
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Check target image exists in ECR
id: check-image
continue-on-error: true
env:
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
run: |
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-$PR_NUMBER-ci
- name: Check target image exists in ECR
id: check-image
continue-on-error: true
run: |
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v7
with:
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
github.rest.issues.createComment({
issue_number: ${{ github.event.inputs.issue_number || github.event.pull_request.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
});
core.setFailed(errMsg);
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v7
with:
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
github.rest.issues.createComment({
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
});
core.setFailed(errMsg);
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-ci
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition
run: |
cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-label.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
- name: Update env vars in the Amazon ECS task definition
run: |
cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
- name: Describe ECS service
id: describe-services
run: |
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
- name: Create ECS service
id: create-service
if: steps.describe-services.outputs.active != 'true'
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
run: |
aws ecs create-service \
--cluster superset-ci \
--service-name pr-$PR_NUMBER-service \
--task-definition superset-ci \
--launch-type FARGATE \
--desired-count 1 \
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=$PR_NUMBER key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
with:
task-definition: ${{ steps.task-def.outputs.task-definition }}
service: pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service
cluster: superset-ci
wait-for-service-stability: true
wait-for-minutes: 10
- name: Describe ECS service
id: describe-services
run: |
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
- name: Create ECS service
id: create-service
if: steps.describe-services.outputs.active != 'true'
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
run: |
aws ecs create-service \
--cluster superset-ci \
--service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service \
--task-definition superset-ci \
--launch-type FARGATE \
--desired-count 1 \
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=${{ github.event.inputs.issue_number || github.event.issue.number }} key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
with:
task-definition: ${{ steps.task-def.outputs.task-definition }}
service: pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service
cluster: superset-ci
wait-for-service-stability: true
wait-for-minutes: 10
- name: List tasks
id: list-tasks
run: |
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
- name: Get network interface
id: get-eni
run: |
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks[0].attachments[0].details | map(select(.name=="networkInterfaceId"))[0].value')" >> $GITHUB_OUTPUT
- name: Get public IP
id: get-ip
run: |
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
script: |
const issue_number = context.payload.inputs?.issue_number || context.issue.number;
github.rest.issues.createComment({
issue_number: issue_number,
owner: context.repo.owner,
repo: context.repo.repo,
body: `@${{ github.actor }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are 'admin'/'admin'. Please allow several minutes for bootstrapping and startup.`
});
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
script: |
const issue_number = context.payload.inputs?.issue_number || context.issue.number;
github.rest.issues.createComment({
issue_number: issue_number,
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})
- name: List tasks
id: list-tasks
run: |
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
- name: Get network interface
id: get-eni
run: |
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
- name: Get public IP
id: get-ip
run: |
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
})
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})

View File

@@ -8,7 +8,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -24,7 +24,7 @@ jobs:
needs: config
if: needs.config.outputs.has-secrets
name: Generate Report
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4

View File

@@ -11,7 +11,7 @@ on:
jobs:
validate-all-ghas:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: Checkout Repository
uses: actions/checkout@v4

View File

@@ -9,7 +9,7 @@ on:
jobs:
superbot-orglabel:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write

View File

@@ -7,7 +7,7 @@ jobs:
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- uses: actions/labeler@v5
with:

View File

@@ -6,7 +6,7 @@ on:
jobs:
latest-release:
name: Add/update tag to new release
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: write

View File

@@ -12,7 +12,7 @@ concurrency:
jobs:
license_check:
name: License Check
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4

View File

@@ -11,7 +11,7 @@ concurrency:
jobs:
check-hold-label:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: Check for 'hold' label
uses: actions/github-script@v7

View File

@@ -10,7 +10,7 @@ on:
jobs:
lint-check:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write

View File

@@ -15,10 +15,10 @@ concurrency:
jobs:
pre-commit:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["current", "previous"]
python-version: ["current", "next", "previous"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
@@ -38,39 +38,12 @@ jobs:
echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >>"${GITHUB_ENV}"
echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >>"${GITHUB_ENV}"
brew install norwoodj/tap/helm-docs
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Install Frontend Dependencies
run: |
cd superset-frontend
npm ci
- name: Install Docs Dependencies
run: |
cd docs
yarn install --immutable
- name: pre-commit
run: |
set +e # Don't exit immediately on failure
export SKIP=eslint-frontend,type-checking-frontend
pre-commit run --all-files
PRE_COMMIT_EXIT_CODE=$?
git diff --quiet --exit-code
GIT_DIFF_EXIT_CODE=$?
if [ "${PRE_COMMIT_EXIT_CODE}" -ne 0 ] || [ "${GIT_DIFF_EXIT_CODE}" -ne 0 ]; then
if [ "${PRE_COMMIT_EXIT_CODE}" -ne 0 ]; then
echo "❌ Pre-commit check failed (exit code: ${EXIT_CODE})."
else
echo "❌ Git working directory is dirty."
echo "📌 This likely means that pre-commit made changes that were not committed."
echo "🔍 Modified files:"
git diff --name-only
fi
if [ $? -ne 0 ] || ! git diff --quiet --exit-code; then
echo "❌ Pre-commit check failed."
echo "🚒 To prevent/address this CI issue, please install/use pre-commit locally."
echo "📖 More details here: https://superset.apache.org/docs/contributing/development#git-hooks"
exit 1

View File

@@ -21,7 +21,7 @@ jobs:
prefer_typescript:
if: github.ref == 'ref/heads/master' && github.event_name == 'pull_request'
name: Prefer TypeScript
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write

View File

@@ -8,7 +8,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -24,7 +24,13 @@ jobs:
needs: config
if: needs.config.outputs.has-secrets
name: Bump version and publish package(s)
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
node-version: [20]
steps:
- uses: actions/checkout@v4
with:
@@ -40,11 +46,11 @@ jobs:
git fetch --prune --unshallow
git tag -d `git tag | grep -E '^trigger-'`
- name: Install Node.js
- name: Use Node.js ${{ matrix.node-version }}
if: env.HAS_TAGS
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: ${{ matrix.node-version }}
- name: Cache npm
if: env.HAS_TAGS

View File

@@ -6,7 +6,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -21,11 +21,12 @@ jobs:
cypress-applitools:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
browser: ["chrome"]
node: [20]
env:
SUPERSET_ENV: development
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -39,7 +40,7 @@ jobs:
APPLITOOLS_BATCH_NAME: Superset Cypress
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -65,7 +66,7 @@ jobs:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: ${{ matrix.node }}
- name: Install npm dependencies
uses: ./.github/actions/cached-dependencies
with:

View File

@@ -12,7 +12,7 @@ env:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -27,7 +27,10 @@ jobs:
cron:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
node: [20]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
@@ -38,7 +41,7 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: ${{ matrix.node }}
- name: Install eyes-storybook dependencies
uses: ./.github/actions/cached-dependencies
with:

View File

@@ -15,7 +15,7 @@ concurrency:
jobs:
test-load-examples:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -23,7 +23,7 @@ jobs:
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset

View File

@@ -12,7 +12,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -28,17 +28,17 @@ jobs:
needs: config
if: needs.config.outputs.has-secrets
name: Build & Deploy
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Set up Node.js
- name: Set up Node.js 20
uses: actions/setup-node@v4
with:
node-version-file: './docs/.nvmrc'
node-version: '20'
- name: Setup Python
uses: ./.github/actions/setup-backend/
- uses: actions/setup-java@v4

View File

@@ -24,10 +24,11 @@ jobs:
- uses: JustinBeckwith/linkinator-action@v1.11.0
continue-on-error: true # This will make the job advisory (non-blocking, no red X)
with:
paths: "**/*.md, **/*.mdx, !superset-frontend/CHANGELOG.md"
paths: "**/*.md, **/*.mdx"
linksToSkip: >-
^https://github.com/apache/(superset|incubator-superset)/(pull|issue)/\d+,
http://localhost:8088/,
docker/.env-non-dev,
http://127.0.0.1:3000/,
http://localhost:9001/,
https://charts.bitnami.com/bitnami,
@@ -50,7 +51,7 @@ jobs:
https://www.plaidcloud.com/
build-deploy:
name: Build & Deploy
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
defaults:
run:
working-directory: docs
@@ -60,10 +61,10 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Set up Node.js
- name: Set up Node.js 20
uses: actions/setup-node@v4
with:
node-version-file: './docs/.nvmrc'
node-version: '20'
- name: yarn install
run: |
yarn install --check-cache

View File

@@ -28,7 +28,6 @@ concurrency:
jobs:
cypress-matrix:
# Somehow one test flakes on 24.04 for unknown reasons, this is the only GHA left on 22.04
runs-on: ubuntu-22.04
permissions:
contents: read
@@ -53,7 +52,7 @@ jobs:
USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true'|| (github.ref == 'refs/heads/master' && 'true') || 'false' }}
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -109,7 +108,7 @@ jobs:
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: "20"
- name: Install npm dependencies
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
@@ -138,7 +137,7 @@ jobs:
run: cypress-run-all ${{ env.USE_DASHBOARD }}
- name: Upload Artifacts
uses: actions/upload-artifact@v4
if: failure()
if: github.event_name == 'workflow_dispatch' && (steps.check.outputs.python || steps.check.outputs.frontend)
with:
path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots
name: cypress-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}-${{ matrix.parallel_id }}
name: cypress-artifact-${{ github.run_id }}-${{ github.job }}

View File

@@ -1,4 +1,4 @@
name: "Frontend Build CI (unit tests, linting & sanity checks)"
name: Frontend
on:
push:
@@ -13,168 +13,68 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
env:
TAG: apache/superset:GHA-${{ github.run_id }}
jobs:
frontend-build:
runs-on: ubuntu-24.04
outputs:
should-run: ${{ steps.check.outputs.frontend }}
runs-on: ubuntu-22.04
steps:
- name: Checkout Code
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check for File Changes
submodules: recursive
- name: Check npm lock file version
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Docker Image
- name: Setup Node.js
if: steps.check.outputs.frontend
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
docker buildx build \
-t $TAG \
--cache-from=type=registry,ref=apache/superset-cache:3.10-slim-bookworm \
--target superset-node-ci \
.
- name: Save Docker Image as Artifact
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install dependencies
if: steps.check.outputs.frontend
run: |
docker save $TAG | gzip > docker-image.tar.gz
- name: Upload Docker Image Artifact
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: eslint
if: steps.check.outputs.frontend
uses: actions/upload-artifact@v4
with:
name: docker-image
path: docker-image.tar.gz
sharded-jest-tests:
needs: frontend-build
if: needs.frontend-build.outputs.should-run == 'true'
strategy:
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8]
fail-fast: false
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v4
with:
name: docker-image
- name: Load Docker Image
run: docker load < docker-image.tar.gz
- name: npm run test with coverage
working-directory: ./superset-frontend
run: |
mkdir -p ${{ github.workspace }}/superset-frontend/coverage
docker run \
-v ${{ github.workspace }}/superset-frontend/coverage:/app/superset-frontend/coverage \
--rm $TAG \
bash -c \
"npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json-summary"
- name: Upload Coverage Artifact
uses: actions/upload-artifact@v4
with:
name: coverage-artifacts-${{ matrix.shard }}
path: superset-frontend/coverage
report-coverage:
needs: [sharded-jest-tests]
if: needs.frontend-build.outputs.should-run == 'true'
runs-on: ubuntu-24.04
steps:
- name: Download Coverage Artifacts
uses: actions/download-artifact@v4
with:
pattern: coverage-artifacts-*
path: coverage/
- name: Show Files
run: find coverage/
- name: Merge Code Coverage
run: npx nyc merge coverage/ merged-output/coverage-summary.json
- name: Upload Code Coverage
npm run eslint -- . --quiet
- name: tsc
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run type
- name: Build plugins packages
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Build plugins Storybook
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build-storybook
- name: superset-ui/core coverage
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run core:cover
- name: unit tests
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run test -- --coverage --silent
# todo: remove this step when fix generator as a project in root jest.config.js
- name: generator-superset unit tests
if: steps.check.outputs.frontend
working-directory: ./superset-frontend/packages/generator-superset
run: npm run test
- name: Upload code coverage
uses: codecov/codecov-action@v5
with:
flags: javascript
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
files: merged-output/coverage-summary.json
slug: apache/superset
core-cover:
needs: frontend-build
if: needs.frontend-build.outputs.should-run == 'true'
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v4
with:
name: docker-image
- name: Load Docker Image
run: docker load < docker-image.tar.gz
- name: superset-ui/core coverage
run: |
docker run --rm $TAG bash -c \
"npm run core:cover"
lint-frontend:
needs: frontend-build
if: needs.frontend-build.outputs.should-run == 'true'
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v4
with:
name: docker-image
- name: Load Docker Image
run: docker load < docker-image.tar.gz
- name: eslint
run: |
docker run --rm $TAG bash -c \
"npm i && npm run eslint -- . --quiet"
- name: tsc
run: |
docker run --rm $TAG bash -c \
"npm run type"
validate-frontend:
needs: frontend-build
if: needs.frontend-build.outputs.should-run == 'true'
runs-on: ubuntu-24.04
steps:
- name: Download Docker Image Artifact
uses: actions/download-artifact@v4
with:
name: docker-image
- name: Load Docker Image
run: docker load < docker-image.tar.gz
- name: Build Plugins Packages
run: |
docker run --rm $TAG bash -c \
"npm run plugins:build"
- name: Build Plugins Storybook
run: |
docker run --rm $TAG bash -c \
"npm run plugins:build-storybook"

View File

@@ -13,7 +13,7 @@ concurrency:
jobs:
lint-test:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
@@ -25,7 +25,7 @@ jobs:
- name: Set up Helm
uses: azure/setup-helm@v4
with:
version: v3.16.4
version: v3.5.4
- name: Setup Python
uses: ./.github/actions/setup-backend/

View File

@@ -20,7 +20,7 @@ on:
jobs:
release:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: write
pull-requests: write

View File

@@ -15,7 +15,7 @@ concurrency:
jobs:
test-mysql:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -74,10 +74,10 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
test-postgres:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["current", "previous"]
python-version: ["current", "next", "previous"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -85,7 +85,7 @@ jobs:
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -136,7 +136,7 @@ jobs:
verbose: true
test-sqlite:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config

View File

@@ -16,7 +16,7 @@ concurrency:
jobs:
test-postgres-presto:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -25,7 +25,7 @@ jobs:
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
@@ -84,7 +84,7 @@ jobs:
verbose: true
test-postgres-hive:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -94,7 +94,7 @@ jobs:
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: postgres:16-alpine
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset

View File

@@ -16,10 +16,10 @@ concurrency:
jobs:
unit-tests:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["previous", "current"]
python-version: ["current", "next"]
env:
PYTHONPATH: ${{ github.workspace }}
steps:

View File

@@ -15,7 +15,7 @@ concurrency:
jobs:
frontend-check-translations:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
@@ -33,7 +33,7 @@ jobs:
if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: '18'
- name: Install dependencies
if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
@@ -46,7 +46,7 @@ jobs:
npm run build-translation
babel-extract:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4

View File

@@ -18,7 +18,7 @@ concurrency:
jobs:
app-checks:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4

View File

@@ -15,7 +15,7 @@ on:
jobs:
supersetbot:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
if: >
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@supersetbot'))

View File

@@ -23,7 +23,7 @@ on:
- 'false'
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -39,26 +39,23 @@ jobs:
needs: config
if: needs.config.outputs.has-secrets
name: docker-release
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
strategy:
matrix:
build_preset: ["dev", "lean", "py310", "websocket", "dockerize", "py311"]
fail-fast: false
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Docker Environment
uses: ./.github/actions/setup-docker
with:
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
install-docker-compose: "false"
build: "true"
- name: Use Node.js 20
uses: actions/setup-node@v4
with:
@@ -67,6 +64,13 @@ jobs:
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
- name: Try to login to DockerHub
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Execute custom Node.js script
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -87,7 +91,6 @@ jobs:
fi
supersetbot docker \
--push \
--preset ${{ matrix.build_preset }} \
--context "$EVENT" \
--context-ref "$RELEASE" $FORCE_LATEST \
@@ -100,7 +103,7 @@ jobs:
update-prs-with-release-info:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write

View File

@@ -8,7 +8,7 @@ on:
jobs:
config:
runs-on: ubuntu-24.04
runs-on: "ubuntu-22.04"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
@@ -23,7 +23,7 @@ jobs:
process-and-upload:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
name: Generate Reports
steps:
- name: Checkout Repository
@@ -32,10 +32,10 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version-file: './superset-frontend/.nvmrc'
node-version: '20'
- name: Install Dependencies
run: npm ci
run: npm install
working-directory: ./superset-frontend
- name: Run Script

View File

@@ -6,7 +6,7 @@ on:
jobs:
welcome:
runs-on: ubuntu-24.04
runs-on: ubuntu-22.04
permissions:
pull-requests: write

8
.gitignore vendored
View File

@@ -21,7 +21,6 @@
*.swp
__pycache__
.aider*
.local
.cache
.bento*
@@ -51,6 +50,7 @@ env
venv*
env_py3
envpy3
env36
local_config.py
/superset_config.py
/superset_text.yml
@@ -66,10 +66,7 @@ superset-websocket/config.json
*.js.map
node_modules
npm-debug.log*
superset/static/assets/*
!superset/static/assets/.gitkeep
superset/static/uploads/*
!superset/static/uploads/.gitkeep
superset/static/assets
superset/static/version_info.json
superset-frontend/**/esm/*
superset-frontend/**/lib/*
@@ -125,4 +122,3 @@ docker/*local*
# Jest test report
test-report.html
superset/static/stats/statistics.html
.aider*

View File

@@ -20,7 +20,7 @@ repos:
hooks:
- id: auto-walrus
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.15.0
rev: v1.13.0
hooks:
- id: mypy
args: [--check-untyped-defs]
@@ -38,6 +38,10 @@ repos:
types-paramiko,
types-Markdown,
]
- repo: https://github.com/peterdemin/pip-compile-multi
rev: v2.6.4
hooks:
- id: pip-compile-multi-verify
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
@@ -60,27 +64,6 @@ repos:
- prettier@3.3.3
args: ["--ignore-path=./superset-frontend/.prettierignore"]
files: "superset-frontend"
- repo: local
hooks:
- id: eslint-frontend
name: eslint (frontend)
entry: ./scripts/eslint.sh
language: system
pass_filenames: true
files: ^superset-frontend/.*\.(js|jsx|ts|tsx)$
- id: eslint-docs
name: eslint (docs)
entry: bash -c 'cd docs && FILES=$(echo "$@" | sed "s|docs/||g") && yarn eslint --fix --ext .js,.jsx,.ts,.tsx --quiet $FILES'
language: system
pass_filenames: true
files: ^docs/.*\.(js|jsx|ts|tsx)$
- id: type-checking-frontend
name: Type-Checking (Frontend)
entry: bash -c './scripts/check-type.js package=superset-frontend excludeDeclarationDir=cypress-base'
language: system
files: ^superset-frontend\/.*\.(js|jsx|ts|tsx)$
exclude: ^superset-frontend/cypress-base\/
require_serial: true
# blacklist unsafe functions like make_url (see #19526)
- repo: https://github.com/skorokithakis/blacklist-pre-commit-hook
rev: e2f070289d8eddcaec0b580d3bde29437e7c8221
@@ -92,11 +75,23 @@ repos:
hooks:
- id: helm-docs
files: helm
verbose: false
args: ["--log-level", "error"]
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.7
rev: v0.8.0
hooks:
- id: ruff
args: [--fix]
args: [ --fix ]
- id: ruff-format
- repo: local
hooks:
- id: pylint
name: pylint
entry: pylint
language: system
types: [python]
exclude: ^(tests/|superset/migrations/|scripts/|RELEASING/|docker/)
args:
[
"-rn", # Only display messages
"-sn", # Don't display the score
"--rcfile=.pylintrc",
]

380
.pylintrc Normal file
View File

@@ -0,0 +1,380 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS,migrations
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
ignore-patterns=
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=superset.extensions.pylint
# Use multiple processes to speed up Pylint.
jobs=2
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=pyarrow
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
enable=
useless-suppression,
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=
cyclic-import, # re-enable once this no longer raises false positives
missing-docstring,
duplicate-code,
line-too-long,
unspecified-encoding,
too-many-instance-attributes # re-enable once this no longer raises false positives
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Tells whether to display a full report or only the messages
reports=yes
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=_,df,ex,f,i,id,j,k,l,o,pk,Run,ts,v,x,y
# Bad variable names which should always be refused, separated by a comma
bad-names=bar,baz,db,fd,foo,sesh,session,tata,toto,tutu
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
property-classes=
abc.abstractproperty,
sqlalchemy.ext.hybrid.hybrid_property
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{1,30}$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct constant names
const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=10
[ELIF]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=100
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=5
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=numpy,pandas,alembic.op,sqlalchemy,alembic.context,flask_appbuilder.security.sqla.PermissionView.role,flask_appbuilder.Model.metadata,flask_appbuilder.Base.metadata
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=contextlib.closing,optparse.Values,thread._local,_thread._local
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators=contextlib.contextmanager
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six.moves,future.builtins
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=10
# Maximum number of branch for function / method body
max-branches=15
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=8
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=optparse
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=builtins.Exception

View File

@@ -18,36 +18,30 @@
######################################################################
# Node stage to deal with static asset construction
######################################################################
ARG PY_VER=3.11.11-slim-bookworm
ARG PY_VER=3.10-slim-bookworm
# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
# Include translations in the final build
ARG BUILD_TRANSLATIONS="false"
######################################################################
# superset-node-ci used as a base for building frontend assets and CI
######################################################################
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node-ci
ARG BUILD_TRANSLATIONS
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
ARG DEV_MODE="false" # Skip frontend build in dev mode
ENV DEV_MODE=${DEV_MODE}
COPY docker/ /app/docker/
# Arguments for build configuration
ARG NPM_BUILD_CMD="build"
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
ARG DEV_MODE="false" # Skip frontend build in dev mode
ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports
ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled
# Install system dependencies required for node-gyp
RUN /app/docker/apt-install.sh build-essential python3 zstd
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh build-essential python3 zstd
# Define environment variables for frontend build
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
# Run the frontend memory monitoring script
RUN /app/docker/frontend-mem-nag.sh
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/frontend-mem-nag.sh
WORKDIR /app/superset-frontend
@@ -56,14 +50,8 @@ RUN mkdir -p /app/superset/static/assets \
/app/superset/translations
# Mount package files and install dependencies if not in dev mode
# NOTE: we mount packages and plugins as they are referenced in package.json as workspaces
# ideally we'd COPY only their package.json. Here npm ci will be cached as long
# as the full content of these folders don't change, yielding a decent cache reuse rate.
# Note that's it's not possible selectively COPY of mount using blobs.
RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
--mount=type=cache,target=/root/.cache \
--mount=type=cache,target=/root/.npm \
if [ "$DEV_MODE" = "false" ]; then \
npm ci; \
else \
@@ -73,183 +61,169 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
# Runs the webpack build process
COPY superset-frontend /app/superset-frontend
######################################################################
# superset-node used for compile frontend assets
######################################################################
FROM superset-node-ci AS superset-node
# Build the frontend if not in dev mode
RUN --mount=type=cache,target=/root/.npm \
if [ "$DEV_MODE" = "false" ]; then \
echo "Running 'npm run ${BUILD_CMD}'"; \
npm run ${BUILD_CMD}; \
else \
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
fi;
# Copy translation files
COPY superset/translations /app/superset/translations
# Build the frontend if not in dev mode
RUN if [ "$DEV_MODE" = "false" ]; then \
BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
else \
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
fi
# Compile .json files from .po translations (if required) and clean up .po files
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
npm run build-translation; \
fi; \
rm -rf /app/superset/translations/*/*/*.po; \
rm -rf /app/superset/translations/*/*/*.mo;
else \
echo "Skipping translations as requested by build flag"; \
fi \
# removing translations files regardless
&& rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
/app/superset/translations/messages.pot
######################################################################
# Base python layer
######################################################################
# Transition to Python base image
FROM python:${PY_VER} AS python-base
ARG SUPERSET_HOME="/app/superset_home"
ENV SUPERSET_HOME=${SUPERSET_HOME}
RUN mkdir -p $SUPERSET_HOME
RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
&& chmod -R 1777 $SUPERSET_HOME \
&& chown -R superset:superset $SUPERSET_HOME
# Some bash scripts needed throughout the layers
COPY --chmod=755 docker/*.sh /app/docker/
RUN pip install --no-cache-dir --upgrade uv
# Using uv as it's faster/simpler than pip
RUN uv venv /app/.venv
ENV PATH="/app/.venv/bin:${PATH}"
RUN pip install --no-cache-dir --upgrade setuptools pip uv
######################################################################
# Python translation compiler layer
# Final lean image...
######################################################################
FROM python-base AS python-translation-compiler
FROM python-base AS lean
ARG BUILD_TRANSLATIONS
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
# Install Python dependencies using docker/pip-install.sh
COPY requirements/translations.txt requirements/
RUN --mount=type=cache,target=/root/.cache/uv \
. /app/.venv/bin/activate && /app/docker/pip-install.sh --requires-build-essential -r requirements/translations.txt
COPY superset/translations/ /app/translations_mo/
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
pybabel compile -d /app/translations_mo | true; \
fi; \
rm -f /app/translations_mo/*/*/*.po; \
rm -f /app/translations_mo/*/*/*.json;
######################################################################
# Python APP common layer
######################################################################
FROM python-base AS python-common
ENV SUPERSET_HOME="/app/superset_home" \
HOME="/app/superset_home" \
SUPERSET_ENV="production" \
FLASK_APP="superset.app:create_app()" \
PYTHONPATH="/app/pythonpath" \
SUPERSET_PORT="8088"
# Copy the entrypoints, make them executable in userspace
COPY --chmod=755 docker/entrypoints /app/docker/entrypoints
# Build argument for including translations
ARG BUILD_TRANSLATIONS="false"
WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
SUPERSET_ENV=production \
FLASK_APP="superset.app:create_app()" \
PYTHONPATH="/app/pythonpath" \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
# Set up necessary directories and user
RUN mkdir -p \
${PYTHONPATH} \
RUN --mount=type=bind,source=./docker,target=/docker \
mkdir -p ${PYTHONPATH} \
superset/static \
requirements \
superset-frontend \
apache_superset.egg-info \
requirements \
&& touch superset/static/version_info.json
# Install Playwright and optionally setup headless browsers
ARG INCLUDE_CHROMIUM="true"
ARG INCLUDE_FIREFOX="false"
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
uv pip install playwright && \
playwright install-deps && \
if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \
if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \
else \
echo "Skipping browser installation"; \
fi
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
&& /docker/apt-install.sh \
curl \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libpq-dev \
libecpg-dev \
libldap2-dev \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
# Copy required files for Python build
COPY pyproject.toml setup.py MANIFEST.in README.md ./
COPY superset-frontend/package.json superset-frontend/
COPY scripts/check-env.py scripts/
# keeping for backward compatibility
COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/
# Some debian libs
RUN /app/docker/apt-install.sh \
curl \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libpq-dev \
libecpg-dev \
libldap2-dev
# Copy compiled things from previous stages
COPY --from=superset-node /app/superset/static/assets superset/static/assets
# TODO, when the next version comes out, use --exclude superset/translations
COPY superset superset
# TODO in the meantime, remove the .po files
RUN rm superset/translations/*/*/*.po
# Merging translations from backend and frontend stages
COPY --from=superset-node /app/superset/translations superset/translations
COPY --from=python-translation-compiler /app/translations_mo superset/translations
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
CMD ["/app/docker/entrypoints/run-server.sh"]
EXPOSE ${SUPERSET_PORT}
######################################################################
# Final lean image...
######################################################################
FROM python-common AS lean
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
COPY --chown=superset:superset requirements/base.txt requirements/
COPY --chown=superset:superset scripts/check-env.py scripts/
# Install Python dependencies using docker/pip-install.sh
COPY requirements/base.txt requirements/
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
/app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
# Install the superset package
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
uv pip install .
RUN python -m compileall /app/superset
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
# Copy the compiled frontend assets from the node image
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
# Copy the main Superset source code
COPY --chown=superset:superset superset superset
# Install Superset itself using docker/pip-install.sh
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh -e .
# Copy .json translations from the node image
COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
# Compile backend translations and clean up
COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
./scripts/translations/generate_mo_files.sh \
&& chown -R superset:superset superset/translations; \
fi \
&& rm -rf superset/translations/messages.pot \
superset/translations/*/LC_MESSAGES/*.po
# Add server run script
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
# Set user and healthcheck
USER superset
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
# Expose port and set CMD
EXPOSE ${SUPERSET_PORT}
CMD ["/usr/bin/run-server.sh"]
######################################################################
# Dev image...
######################################################################
FROM python-common AS dev
FROM lean AS dev
# Debian libs needed for dev
RUN /app/docker/apt-install.sh \
git \
pkg-config \
default-libmysqlclient-dev
USER root
# Install dev dependencies
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
libx11-xcb1 \
libasound2 \
libxtst6 \
git \
pkg-config
# Install Playwright and its dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
uv pip install --system playwright \
&& playwright install-deps
# Optionally install Chromium
RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
playwright install chromium; \
else \
echo "Skipping Chromium installation in dev mode"; \
fi
# Install GeckoDriver WebDriver and Firefox (if required)
ARG GECKODRIVER_VERSION=v0.34.0
ARG FIREFOX_VERSION=125.0.3
RUN --mount=type=bind,source=./docker,target=/docker \
if [ "$INCLUDE_FIREFOX" = "true" ]; then \
/docker/apt-install.sh wget bzip2 \
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
else \
echo "Skipping Firefox installation in dev mode"; \
fi
# Install MySQL client dependencies
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh default-libmysqlclient-dev
# Copy development requirements and install them
COPY requirements/*.txt requirements/
# Install Python dependencies using docker/pip-install.sh
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
/app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt
# Install the superset package
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
uv pip install .
RUN uv pip install .[postgres]
RUN python -m compileall /app/superset
COPY --chown=superset:superset requirements/development.txt requirements/
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh --requires-build-essential -r requirements/development.txt
USER superset
@@ -257,7 +231,7 @@ USER superset
# CI image...
######################################################################
FROM lean AS ci
USER root
RUN uv pip install .[postgres]
USER superset
CMD ["/app/docker/entrypoints/docker-ci.sh"]
COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
CMD ["/app/docker/docker-ci.sh"]

View File

@@ -87,6 +87,9 @@ format: py-format js-format
py-format: pre-commit
pre-commit run black --all-files
py-lint: pre-commit
pylint -j 0 superset
js-format:
cd superset-frontend; npm run prettier

View File

@@ -73,8 +73,7 @@ Superset provides:
**Video Overview**
<!-- File hosted here https://github.com/apache/superset-site/raw/lfs/superset-video-4k.mp4 -->
[superset-video-1080p.webm](https://github.com/user-attachments/assets/b37388f7-a971-409c-96a7-90c4e31322e6)
[superset-video-4k.webm](https://github.com/apache/superset/assets/812905/da036bc2-150c-4ee7-80f9-75e63210ff76)
<br/>
@@ -138,7 +137,6 @@ Here are some of the major database solutions that are supported:
<img src="https://superset.apache.org/img/databases/sap-hana.png" alt="oceanbase" border="0" width="220" />
<img src="https://superset.apache.org/img/databases/denodo.png" alt="denodo" border="0" width="200" />
<img src="https://superset.apache.org/img/databases/ydb.svg" alt="ydb" border="0" width="200" />
<img src="https://superset.apache.org/img/databases/tdengine.png" alt="TDengine" border="0" width="200" />
</p>
**A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/configuration/databases).
@@ -147,7 +145,7 @@ Want to add support for your datastore or data engine? Read more [here](https://
## Installation and Configuration
Try out Superset's [quickstart](https://superset.apache.org/docs/quickstart/) guide or learn about [the options for production deployments](https://superset.apache.org/docs/installation/architecture/).
[Extended documentation for Superset](https://superset.apache.org/docs/installation/docker-compose)
## Get Involved

View File

@@ -30,12 +30,12 @@ RUN apt-get install -y apt-transport-https apt-utils
# Install superset dependencies
# https://superset.apache.org/docs/installation/installing-superset-from-scratch
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium zstd
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
# Install nodejs for custom build
# https://nodejs.org/en/download/package-manager/
RUN set -eux; \
curl -sL https://deb.nodesource.com/setup_20.x | bash -; \
curl -sL https://deb.nodesource.com/setup_18.x | bash -; \
apt-get install -y nodejs; \
node --version;
RUN if ! which npm; then apt-get install -y npm; fi
@@ -64,7 +64,7 @@ RUN pip install --upgrade setuptools pip \
RUN flask fab babel-compile --target superset/translations
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/ \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH \
SUPERSET_TESTENV=true
COPY from_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -29,16 +29,13 @@ RUN apt-get install -y apt-transport-https apt-utils
# Install superset dependencies
# https://superset.apache.org/docs/installation/installing-superset-from-scratch
RUN apt-get install -y subversion build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium zstd
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
# Install nodejs for custom build
# https://nodejs.org/en/download/package-manager/
RUN set -eux; \
curl -sL https://deb.nodesource.com/setup_20.x | bash -; \
apt-get install -y nodejs; \
node --version;
RUN if ! which npm; then apt-get install -y npm; fi
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
&& apt-get install -y nodejs
RUN mkdir -p /home/superset
RUN chown superset /home/superset
@@ -49,12 +46,14 @@ ARG VERSION
# Can fetch source from svn or copy tarball from local mounted directory
RUN svn co https://dist.apache.org/repos/dist/dev/superset/$VERSION ./
RUN tar -xvf *.tar.gz
WORKDIR /home/superset/apache-superset-$VERSION/superset-frontend
WORKDIR apache-superset-$VERSION
RUN npm ci \
RUN cd superset-frontend \
&& npm ci \
&& npm run build \
&& rm -rf node_modules
WORKDIR /home/superset/apache-superset-$VERSION
RUN pip install --upgrade setuptools pip \
&& pip install -r requirements/base.txt \
@@ -63,6 +62,6 @@ RUN pip install --upgrade setuptools pip \
RUN flask fab babel-compile --target superset/translations
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/
PYTHONPATH=/home/superset/superset/:$PYTHONPATH
COPY from_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -454,11 +454,8 @@ cd ../
# Compile translations for the backend
./scripts/translations/generate_po_files.sh
# update build version number
sed -i '' "s/version_string = .*/version_string = \"$SUPERSET_VERSION\"/" setup.py
# build the python distribution
python setup.py sdist
python -m build
```
Publish to PyPI

View File

@@ -232,7 +232,8 @@ class GitChangeLog:
for log in self._logs:
yield {
"pr_number": log.pr_number,
"pr_link": f"https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}",
"pr_link": f"https://github.com/{SUPERSET_REPO}/pull/"
f"{log.pr_number}",
"message": log.message,
"time": log.time,
"author": log.author,
@@ -271,14 +272,14 @@ class GitLogs:
@staticmethod
def _git_get_current_head() -> str:
output = os.popen("git status | head -1").read() # noqa: S605, S607
output = os.popen("git status | head -1").read()
match = re.match("(?:HEAD detached at|On branch) (.*)", output)
if not match:
return ""
return match.group(1)
def _git_checkout(self, git_ref: str) -> None:
os.popen(f"git checkout {git_ref}").read() # noqa: S605
os.popen(f"git checkout {git_ref}").read()
current_head = self._git_get_current_head()
if current_head != git_ref:
print(f"Could not checkout {git_ref}")
@@ -289,7 +290,7 @@ class GitLogs:
current_git_ref = self._git_get_current_head()
self._git_checkout(self._git_ref)
output = (
os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"') # noqa: S605, S607
os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
.read()
.split("\n")
)
@@ -322,9 +323,9 @@ class BaseParameters:
def print_title(message: str) -> None:
print(f"{50 * '-'}")
print(f"{50*'-'}")
print(message)
print(f"{50 * '-'}")
print(f"{50*'-'}")
@click.group()
@@ -348,14 +349,14 @@ def compare(base_parameters: BaseParameters) -> None:
previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs
print_title(
f"Pull requests from {current_logs.git_ref} not in {previous_logs.git_ref}"
f"Pull requests from " f"{current_logs.git_ref} not in {previous_logs.git_ref}"
)
previous_diff_logs = previous_logs.diff(current_logs)
for diff_log in previous_diff_logs:
print(f"{diff_log}")
print_title(
f"Pull requests from {previous_logs.git_ref} not in {current_logs.git_ref}"
f"Pull requests from " f"{previous_logs.git_ref} not in {current_logs.git_ref}"
)
current_diff_logs = current_logs.diff(previous_logs)
for diff_log in current_diff_logs:

View File

@@ -31,7 +31,7 @@ except ModuleNotFoundError:
RECEIVER_EMAIL = "dev@superset.apache.org"
PROJECT_NAME = "Superset"
PROJECT_MODULE = "superset"
PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application." # noqa: E501
PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application."
def string_comma_to_list(message: str) -> list[str]:

View File

@@ -23,12 +23,12 @@ from typing import Optional
import requests
# Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512` # noqa: E501
# Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512`
def get_sha512_hash(filename: str) -> str:
"""Run the shasum command on the file and return the SHA512 hash."""
result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE) # noqa: S603, S607
result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE)
sha512_hash = result.stdout.decode().split()[0]
return sha512_hash
@@ -43,7 +43,7 @@ def read_sha512_file(filename: str) -> str:
def verify_sha512(filename: str) -> str:
"""Verify if the SHA512 hash of the file matches with the hash in the .sha512 file.""" # noqa: E501
"""Verify if the SHA512 hash of the file matches with the hash in the .sha512 file."""
sha512_hash = get_sha512_hash(filename)
sha512_file_content = read_sha512_file(filename)
@@ -53,15 +53,14 @@ def verify_sha512(filename: str) -> str:
return "SHA failed"
# Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file # noqa: E501
# Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file
def get_gpg_info(filename: str) -> tuple[Optional[str], Optional[str]]:
"""Run the GPG verify command and extract RSA key and email address."""
asc_filename = filename + ".asc"
result = subprocess.run( # noqa: S603
["gpg", "--verify", asc_filename, filename], # noqa: S607
capture_output=True, # noqa: S607
result = subprocess.run(
["gpg", "--verify", asc_filename, filename], capture_output=True
)
output = result.stderr.decode()
@@ -91,7 +90,7 @@ def get_gpg_info(filename: str) -> tuple[Optional[str], Optional[str]]:
def verify_key(key: str, email: Optional[str]) -> str:
"""Fetch the KEYS file and verify if the RSA/EDDSA key and email match."""
url = "https://downloads.apache.org/superset/KEYS"
response = requests.get(url) # noqa: S113
response = requests.get(url)
if response.status_code == 200:
if key not in response.text:
return "RSA/EDDSA key not found on KEYS page"

View File

@@ -44,11 +44,12 @@ These features are **finished** but currently being tested. They are usable, but
- ALLOW_FULL_CSV_EXPORT
- CACHE_IMPERSONATION
- CONFIRM_DASHBOARD_DIFF
- DRILL_TO_DETAIL
- DYNAMIC_PLUGINS
- DATE_FORMAT_IN_EMAIL_SUBJECT: [(docs)](https://superset.apache.org/docs/configuration/alerts-reports#commons)
- ENABLE_SUPERSET_META_DB: [(docs)](https://superset.apache.org/docs/configuration/databases/#querying-across-databases)
- ESTIMATE_QUERY_COST
- GLOBAL_ASYNC_QUERIES [(docs)](https://github.com/apache/superset/blob/master/CONTRIBUTING.md#async-chart-queries)
- HORIZONTAL_FILTER_BAR
- IMPERSONATE_WITH_EMAIL_PREFIX
- PLAYWRIGHT_REPORTS_AND_THUMBNAILS
- RLS_IN_SQLLAB
@@ -62,8 +63,9 @@ These features flags are **safe for production**. They have been tested and will
[//]: # "PLEASE KEEP THESE LISTS SORTED ALPHABETICALLY"
### Flags on the path to feature launch and flag deprecation/removal
- DASHBOARD_VIRTUALIZATION
- DRILL_BY
- DISABLE_LEGACY_DATASOURCE_EDITOR
### Flags retained for runtime configuration
@@ -77,7 +79,6 @@ independently. This new framework will also allow for non-boolean configurations
- ALLOW_ADHOC_SUBQUERY
- DASHBOARD_RBAC [(docs)](https://superset.apache.org/docs/using-superset/creating-your-first-dashboard#manage-access-to-dashboards)
- DATAPANEL_CLOSED_BY_DEFAULT
- DRILL_BY
- DRUID_JOINS
- EMBEDDABLE_CHARTS
- EMBEDDED_SUPERSET
@@ -97,6 +98,6 @@ These features flags currently default to True and **will be removed in a future
[//]: # "PLEASE KEEP THE LIST SORTED ALPHABETICALLY"
- AVOID_COLORS_COLLISION
- DRILL_TO_DETAIL
- DASHBOARD_CROSS_FILTERS
- ENABLE_JAVASCRIPT_CONTROLS
- KV_STORE

View File

@@ -25,8 +25,8 @@ all you have to do is file a simple PR [like this one](https://github.com/apache
the categorization is inaccurate, please file a PR with your correction as well.
Join our growing community!
### Sharing Economy
### Sharing Economy
- [Airbnb](https://github.com/airbnb)
- [Faasos](https://faasos.com/) [@shashanksingh]
- [Free2Move](https://www.free2move.com/) [@PaoloTerzi]
@@ -36,7 +36,6 @@ Join our growing community!
- [Ontruck](https://www.ontruck.com/)
### Financial Services
- [Aktia Bank plc](https://www.aktia.com)
- [American Express](https://www.americanexpress.com) [@TheLastSultan]
- [bumper](https://www.bumper.co/) [@vasu-ram, @JamiePercival]
@@ -44,53 +43,43 @@ Join our growing community!
- [Capital Service S.A.](https://capitalservice.pl) [@pkonarzewski]
- [Clark.de](https://clark.de/)
- [KarrotPay](https://www.daangnpay.com/)
- [Remita](https://remita.net) [@mujibishola]
- [Taveo](https://www.taveo.com) [@codek]
- [Unit](https://www.unit.co/about-us) [@amitmiran137]
- [Wise](https://wise.com) [@koszti]
- [Xendit](https://xendit.co/) [@LieAlbertTriAdrian]
- [Cover Genius](https://covergenius.com/)
### Gaming
- [Popoko VM Games Studio](https://popoko.live)
### E-Commerce
- [AiHello](https://www.aihello.com) [@ganeshkrishnan1]
- [Bazaar Technologies](https://www.bazaartech.com) [@umair-abro]
- [Dragonpass](https://www.dragonpass.com.cn/) [@zhxjdwh]
- [Dropit Shopping](https://www.dropit.shop/) [@dropit-dev]
- [Fanatics](https://www.fanatics.com/) [@coderfender]
- [Fordeal](https://www.fordeal.com) [@Renkai]
- [Fynd](https://www.fynd.com/) [@darpanjain07]
- [GFG - Global Fashion Group](https://global-fashion-group.com) [@ksaagariconic]
- [GoTo/Gojek](https://www.gojek.io/) [@gwthm-in]
- [HuiShouBao](https://www.huishoubao.com/) [@Yukinoshita-Yukino]
- [Now](https://www.now.vn/) [@davidkohcw]
- [Qunar](https://www.qunar.com/) [@flametest]
- [Rakuten Viki](https://www.viki.com)
- [Shopee](https://shopee.sg) [@xiaohanyu]
- [Shopkick](https://www.shopkick.com) [@LAlbertalli]
- [ShopUp](https://www.shopup.org/) [@gwthm-in]
- [Tails.com](https://tails.com/gb/) [@alanmcruickshank]
- [THE ICONIC](https://theiconic.com.au/) [@ksaagariconic]
- [Utair](https://www.utair.ru) [@utair-digital]
- [VkusVill](https://vkusvill.ru/) [@ETselikov]
- [Zalando](https://www.zalando.com) [@dmigo]
- [Zalora](https://www.zalora.com) [@ksaagariconic]
- [Zepto](https://www.zeptonow.com/) [@gwthm-in]
### Enterprise Technology
- [A3Data](https://a3data.com.br) [@neylsoncrepalde]
- [Analytics Aura](https://analyticsaura.com/) [@Analytics-Aura]
- [Apollo GraphQL](https://www.apollographql.com/) [@evans]
- [Astronomer](https://www.astronomer.io) [@ryw]
- [Avesta Technologies](https://avestatechnologies.com/) [@TheRum]
- [Caizin](https://caizin.com/) [@tejaskatariya]
- [Canonical](https://canonical.com)
- [Careem](https://www.careem.com/) [@samraHanif0340]
- [Careem](https://www.careem.com/) [@SamraHanifCareem]
- [Cloudsmith](https://cloudsmith.io) [@alancarson]
- [Cyberhaven](https://www.cyberhaven.com/) [@toliver-ch]
- [Deepomatic](https://deepomatic.com/) [@Zanoellia]
@@ -121,11 +110,8 @@ Join our growing community!
- [PubNub](https://pubnub.com) [@jzucker2]
- [ReadyTech](https://www.readytech.io)
- [Reward Gateway](https://www.rewardgateway.com)
- [RIADVICE](https://riadvice.tn) [@riadvice]
- [ScopeAI](https://www.getscopeai.com) [@iloveluce]
- [shipmnts](https://shipmnts.com)
- [Showmax](https://showmax.com) [@bobek]
- [SingleStore](https://www.singlestore.com/)
- [TechAudit](https://www.techaudit.info) [@ETselikov]
- [Tenable](https://www.tenable.com) [@dflionis]
- [Tentacle](https://www.linkedin.com/company/tentacle-cmi/) [@jdclarke5]
@@ -133,13 +119,11 @@ Join our growing community!
- [Tobii](https://www.tobii.com/) [@dwa]
- [Tooploox](https://www.tooploox.com/) [@jakubczaplicki]
- [Unvired](https://unvired.com) [@srinisubramanian]
- [Virtuoso QA](https://www.virtuosoqa.com)
- [Whale](https://whale.im)
- [Windsor.ai](https://www.windsor.ai/) [@octaviancorlade]
- [Zeta](https://www.zeta.tech/) [@shaikidris]
### Media & Entertainment
- [6play](https://www.6play.fr) [@CoryChaplin]
- [bilibili](https://www.bilibili.com) [@Moinheart]
- [BurdaForward](https://www.burda-forward.de/en/)
@@ -152,10 +136,8 @@ Join our growing community!
- [Zaihang](https://www.zaih.com/)
### Education
- [Aveti Learning](https://avetilearning.com/) [@TheShubhendra]
- [Brilliant.org](https://brilliant.org/)
- [Open edX](https://openedx.org/)
- [Platzi.com](https://platzi.com/)
- [Sunbird](https://www.sunbird.org/) [@eksteporg]
- [The GRAPH Network](https://thegraphnetwork.org/) [@fccoelho]
@@ -164,7 +146,6 @@ Join our growing community!
- [WikiMedia Foundation](https://wikimediafoundation.org) [@vg]
### Energy
- [Airboxlab](https://foobot.io) [@antoine-galataud]
- [DouroECI](https://www.douroeci.com/) [@nunohelibeires]
- [Safaricom](https://www.safaricom.co.ke/) [@mmutiso]
@@ -172,7 +153,6 @@ Join our growing community!
- [Wattbewerb](https://wattbewerb.de/) [@wattbewerb]
### Healthcare
- [Amino](https://amino.com) [@shkr]
- [Bluesquare](https://www.bluesquarehub.com/) [@madewulf]
- [Care](https://www.getcare.io/) [@alandao2021]
@@ -185,29 +165,24 @@ Join our growing community!
- [2070Health](https://2070health.com/)
### HR / Staffing
- [Swile](https://www.swile.co/) [@PaoloTerzi]
- [Symmetrics](https://www.symmetrics.fyi)
- [bluquist](https://bluquist.com/)
### Government
### Government / Non-Profit
- [City of Ann Arbor, MI](https://www.a2gov.org/) [@sfirke]
- [RIS3 Strategy of CZ, MIT CR](https://www.ris3.cz/) [@RIS3CZ]
- [NRLM - Sarathi, India](https://pib.gov.in/PressReleasePage.aspx?PRID=1999586)
### Travel
- [Agoda](https://www.agoda.com/) [@lostseaway, @maiake, @obombayo]
- [Skyscanner](https://www.skyscanner.net/) [@cleslie, @stanhoucke]
### Others
- [10Web](https://10web.io/)
- [AI inside](https://inside.ai/en/)
- [Automattic](https://automattic.com/) [@Khrol, @Usiel]
- [Dropbox](https://www.dropbox.com/) [@bkyryliuk]
- [Flowbird](https://flowbird.com) [@EmmanuelCbd]
- [GEOTAB](https://www.geotab.com) [@JZ6]
- [Grassroot](https://www.grassrootinstitute.org/)
- [Increff](https://www.increff.com/) [@ishansinghania]

View File

@@ -43,8 +43,8 @@ under the License.
| can this form post on ResetPasswordView |:heavy_check_mark:|O|O|O|
| can this form get on ResetMyPasswordView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can this form post on ResetMyPasswordView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can this form get on UserInfoEditView |:heavy_check_mark:|O|O|O|
| can this form post on UserInfoEditView |:heavy_check_mark:|O|O|O|
| can this form get on UserInfoEditView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can this form post on UserInfoEditView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can show on UserDBModelView |:heavy_check_mark:|O|O|O|
| can edit on UserDBModelView |:heavy_check_mark:|O|O|O|
| can delete on UserDBModelView |:heavy_check_mark:|O|O|O|
@@ -65,6 +65,7 @@ under the License.
| can get on MenuApi |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can list on AsyncEventsRestApi |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can invalidate on CacheRestApi |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can function names on Database |:heavy_check_mark:|O|O|O|
| can csv upload on Database |:heavy_check_mark:|O|O|O|
| can excel upload on Database |:heavy_check_mark:|O|O|O|
| can query form data on Api |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
@@ -75,6 +76,7 @@ under the License.
| can get on Datasource |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can my queries on SqlLab |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
| can log on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can schemas access for csv upload on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can import dashboards on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can schemas on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can sqllab history on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
@@ -116,6 +118,8 @@ under the License.
| menu access on Data |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Databases |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Datasets |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Upload a CSV |:heavy_check_mark:|:heavy_check_mark:|O|O|
| menu access on Upload Excel |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Charts |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Dashboards |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on SQL Lab |:heavy_check_mark:|O|O|:heavy_check_mark:|
@@ -125,6 +129,13 @@ under the License.
| all datasource access on all_datasource_access |:heavy_check_mark:|:heavy_check_mark:|O|O|
| all database access on all_database_access |:heavy_check_mark:|:heavy_check_mark:|O|O|
| all query access on all_query_access |:heavy_check_mark:|O|O|O|
| can edit on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| can list on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| can show on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| can userinfo on UserOAuthModelView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can add on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| can delete on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| userinfoedit on UserOAuthModelView |:heavy_check_mark:|O|O|O|
| can write on DynamicPlugin |:heavy_check_mark:|O|O|O|
| can edit on DynamicPlugin |:heavy_check_mark:|O|O|O|
| can list on DynamicPlugin |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
@@ -181,6 +192,7 @@ under the License.
| can share chart on Superset |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can this form get on ColumnarToDatabaseView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can this form post on ColumnarToDatabaseView |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| menu access on Upload a Columnar file |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can export on Chart |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can write on DashboardFilterStateRestApi |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|
| can read on DashboardFilterStateRestApi |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|O|

View File

@@ -24,30 +24,11 @@ assists people when migrating to a new version.
## Next
- [32317](https://github.com/apache/superset/pull/32317) The horizontal filter bar feature is now out of testing/beta development and its feature flag `HORIZONTAL_FILTER_BAR` has been removed.
- [31976](https://github.com/apache/superset/pull/31976) Removed the `DISABLE_LEGACY_DATASOURCE_EDITOR` feature flag. The previous value of the feature flag was `True` and now the feature is permanently removed.
- [31959](https://github.com/apache/superset/pull/32000) Removes CSV_UPLOAD_MAX_SIZE config, use your web server to control file upload size.
- [31959](https://github.com/apache/superset/pull/31959) Removes the following endpoints from data uploads: `/api/v1/database/<id>/<file type>_upload` and `/api/v1/database/<file type>_metadata`, in favour of new one (Details on the PR). And simplifies permissions.
- [31844](https://github.com/apache/superset/pull/31844) The `ALERT_REPORTS_EXECUTE_AS` and `THUMBNAILS_EXECUTE_AS` config parameters have been renamed to `ALERT_REPORTS_EXECUTORS` and `THUMBNAILS_EXECUTORS` respectively. A new config flag `CACHE_WARMUP_EXECUTORS` has also been introduced to be able to control which user is used to execute cache warmup tasks. Finally, the config flag `THUMBNAILS_SELENIUM_USER` has been removed. To use a fixed executor for async tasks, use the new `FixedExecutor` class. See the config and docs for more info on setting up different executor profiles.
- [31894](https://github.com/apache/superset/pull/31894) Domain sharding is deprecated in favor of HTTP2. The `SUPERSET_WEBSERVER_DOMAINS` configuration will be removed in the next major version (6.0)
- [31794](https://github.com/apache/superset/pull/31794) Removed the previously deprecated `DASHBOARD_CROSS_FILTERS` feature flag
- [31774](https://github.com/apache/superset/pull/31774): Fixes the spelling of the `USE-ANALAGOUS-COLORS` feature flag. Please update any scripts/configuration item to use the new/corrected `USE-ANALOGOUS-COLORS` flag spelling.
- [31582](https://github.com/apache/superset/pull/31582) Removed the legacy Area, Bar, Event Flow, Heatmap, Histogram, Line, Sankey, and Sankey Loop charts. They were all automatically migrated to their ECharts counterparts with the exception of the Event Flow and Sankey Loop charts which were removed as they were not actively maintained and not widely used. If you were using the Event Flow or Sankey Loop charts, you will need to find an alternative solution.
- [31198](https://github.com/apache/superset/pull/31198) Disallows by default the use of the following ClickHouse functions: "version", "currentDatabase", "hostName".
- [29798](https://github.com/apache/superset/pull/29798) Since 3.1.0, the intial schedule for an alert or report was mistakenly offset by the specified timezone's relation to UTC. The initial schedule should now begin at the correct time.
- [30021](https://github.com/apache/superset/pull/30021) The `dev` layer in our Dockerfile no long includes firefox binaries, only Chromium to reduce bloat/docker-build-time.
- [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`.
- [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before.
- [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context.
- [31413](https://github.com/apache/superset/pull/31413) Enable the DATE_FORMAT_IN_EMAIL_SUBJECT feature flag to allow users to specify a date format for the email subject, which will then be replaced with the actual date.
- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ...
- [31503](https://github.com/apache/superset/pull/31503) Deprecating python 3.9.x support, 3.11 is now the recommended version and 3.10 is still supported over the Superset 5.0 lifecycle.
- [29121](https://github.com/apache/superset/pull/29121) Removed the `css`, `position_json`, and `json_metadata` from the payload of the dashboard list endpoint (`GET api/v1/dashboard`) for performance reasons.
- [29163](https://github.com/apache/superset/pull/29163) Removed the `SHARE_QUERIES_VIA_KV_STORE` and `KV_STORE` feature flags and changed the way Superset shares SQL Lab queries to use permalinks. The legacy `/kv` API was removed but we still support legacy links in 5.0. In 6.0, only permalinks will be supported.
- [25166](https://github.com/apache/superset/pull/25166) Changed the default configuration of `UPLOAD_FOLDER` from `/app/static/uploads/` to `/static/uploads/`. It also removed the unused `IMG_UPLOAD_FOLDER` and `IMG_UPLOAD_URL` configuration options.
- [30284](https://github.com/apache/superset/pull/30284) Deprecated GLOBAL_ASYNC_QUERIES_REDIS_CONFIG in favor of the new GLOBAL_ASYNC_QUERIES_CACHE_BACKEND configuration. To leverage Redis Sentinel, set CACHE_TYPE to RedisSentinelCache, or use RedisCache for standalone Redis
- [31961](https://github.com/apache/superset/pull/31961) Upgraded React from version 16.13.1 to 17.0.2. If you are using custom frontend extensions or plugins, you may need to update them to be compatible with React 17.
- [31260](https://github.com/apache/superset/pull/31260) Docker images now use `uv pip install` instead of `pip install` to manage the python envrionment. Most docker-based deployments will be affected, whether you derive one of the published images, or have custom bootstrap script that install python libraries (drivers)
### Potential Downtime

View File

@@ -22,6 +22,9 @@
# unique random secure passwords and SECRET_KEY.
# -----------------------------------------------------------------------
x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev}
x-superset-depends-on: &superset-depends-on
- db
- redis
x-superset-volumes:
&superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@@ -41,7 +44,7 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
image: postgres:16
image: postgres:15
container_name: superset_db
restart: unless-stopped
volumes:
@@ -61,12 +64,8 @@ services:
restart: unless-stopped
ports:
- 8088:8088
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
volumes: *superset-volumes
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-init:
image: *superset-image
@@ -77,18 +76,11 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
depends_on:
db:
condition: service_started
redis:
condition: service_started
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
disable: true
environment:
SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-worker:
image: *superset-image
@@ -100,9 +92,7 @@ services:
- path: docker/.env-local # optional override
required: false
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
@@ -111,8 +101,6 @@ services:
"CMD-SHELL",
"celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
]
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-worker-beat:
image: *superset-image
@@ -124,15 +112,11 @@ services:
- path: docker/.env-local # optional override
required: false
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
disable: true
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
volumes:
superset_home:

View File

@@ -21,6 +21,9 @@
# create you own docker environment file (docker/.env) with your own
# unique random secure passwords and SECRET_KEY.
# -----------------------------------------------------------------------
x-superset-depends-on: &superset-depends-on
- db
- redis
x-superset-volumes:
&superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@@ -46,7 +49,7 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
image: postgres:16
image: postgres:15
container_name: superset_db
restart: unless-stopped
volumes:
@@ -67,12 +70,8 @@ services:
restart: unless-stopped
ports:
- 8088:8088
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
volumes: *superset-volumes
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-init:
container_name: superset_init
@@ -84,18 +83,11 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
depends_on:
db:
condition: service_started
redis:
condition: service_started
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
disable: true
environment:
SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-worker:
build:
@@ -108,9 +100,7 @@ services:
- path: docker/.env-local # optional override
required: false
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
@@ -119,8 +109,6 @@ services:
"CMD-SHELL",
"celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
]
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-worker-beat:
build:
@@ -133,15 +121,11 @@ services:
- path: docker/.env-local # optional override
required: false
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
healthcheck:
disable: true
environment:
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
volumes:
superset_home:

View File

@@ -22,6 +22,9 @@
# unique random secure passwords and SECRET_KEY.
# -----------------------------------------------------------------------
x-superset-user: &superset-user root
x-superset-depends-on: &superset-depends-on
- db
- redis
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@@ -32,14 +35,11 @@ x-superset-volumes: &superset-volumes
x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
target: dev
cache_from:
- apache/superset-cache:3.10-slim-bookworm
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
INCLUDE_FIREFOX: ${INCLUDE_FIREFOX:-false}
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
services:
nginx:
@@ -67,7 +67,7 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
image: postgres:16
image: postgres:15
container_name: superset_db
restart: unless-stopped
ports:
@@ -89,18 +89,13 @@ services:
restart: unless-stopped
ports:
- 8088:8088
# When in cypress-mode ->
- 8081:8081
extra_hosts:
- "host.docker.internal:host-gateway"
user: *superset-user
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
volumes: *superset-volumes
environment:
CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-websocket:
container_name: superset_websocket
@@ -145,17 +140,11 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
depends_on:
db:
condition: service_started
redis:
condition: service_started
depends_on: *superset-depends-on
user: *superset-user
volumes: *superset-volumes
environment:
CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
healthcheck:
disable: true
@@ -168,17 +157,12 @@ services:
# and build it on startup while firing docker-frontend.sh in dev mode, where
# it'll mount and watch local files and rebuild as you update them
DEV_MODE: "true"
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
environment:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
NPM_RUN_PRUNE: false
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
# configuring the dev-server to use the host.docker.internal to connect to the backend
superset: "http://superset:8088"
ports:
- "127.0.0.1:9000:9000" # exposing the dynamic webpack dev server
container_name: superset_node
command: ["/app/docker/docker-frontend.sh"]
env_file:
@@ -186,6 +170,7 @@ services:
required: true
- path: docker/.env-local # optional override
required: false
depends_on: *superset-depends-on
volumes: *superset-volumes
superset-worker:
@@ -200,12 +185,8 @@ services:
required: false
environment:
CELERYD_CONCURRENCY: 2
CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: *superset-user
volumes: *superset-volumes
extra_hosts:
@@ -227,15 +208,11 @@ services:
- path: docker/.env-local # optional override
required: false
restart: unless-stopped
depends_on:
- superset-worker
depends_on: *superset-depends-on
user: *superset-user
volumes: *superset-volumes
healthcheck:
disable: true
environment:
CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
superset-tests-worker:
build:
@@ -256,11 +233,8 @@ services:
REDIS_RESULTS_DB: 3
REDIS_HOST: localhost
CELERYD_CONCURRENCY: 8
SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
network_mode: host
depends_on:
superset-init:
condition: service_completed_successfully
depends_on: *superset-depends-on
user: *superset-user
volumes: *superset-volumes
healthcheck:

View File

@@ -15,11 +15,8 @@
# limitations under the License.
#
# Allowing python to print() in docker
PYTHONUNBUFFERED=1
COMPOSE_PROJECT_NAME=superset
DEV_MODE=true
# database configurations (do not modify)
DATABASE_DB=superset
@@ -66,4 +63,3 @@ SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
ENABLE_PLAYWRIGHT=false
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
SUPERSET_LOG_LEVEL=info

View File

@@ -68,7 +68,7 @@ Don't forget to reload the page to take the new frontend into account though.
## Production
It is possible to run Superset in non-development mode by using [`docker-compose-non-dev.yml`](../docker-compose-non-dev.yml). This file excludes the volumes needed for development.
It is possible to run Superset in non-development mode by using [`docker-compose-non-dev.yml`](../docker-compose-non-dev.yml). This file excludes the volumes needed for development and uses [`./docker/.env-non-dev`](./.env-non-dev) which sets the variable `SUPERSET_ENV` to `production`.
## Resource Constraints

View File

@@ -18,39 +18,19 @@
set -eo pipefail
# Make python interactive
if [ "$DEV_MODE" == "true" ]; then
if [ "$(whoami)" = "root" ] && command -v uv > /dev/null 2>&1; then
echo "Reinstalling the app in editable mode"
uv pip install -e .
fi
fi
REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
PORT=${PORT:-8088}
# If Cypress run overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export POSTGRES_DB=superset_cypress
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset_cypress
PORT=8081
fi
if [[ "$DATABASE_DIALECT" == postgres* ]] && [ "$(whoami)" = "root" ]; then
# older images may not have the postgres dev requirements installed
echo "Installing postgres requirements"
if command -v uv > /dev/null 2>&1; then
# Use uv in newer images
uv pip install -e .[postgres]
else
# Use pip in older images
pip install -e .[postgres]
fi
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
fi
#
# Make sure we have dev requirements installed
#
if [ -f "${REQUIREMENTS_LOCAL}" ]; then
echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
uv pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
else
echo "Skipping local overrides"
fi
@@ -68,7 +48,7 @@ case "${1}" in
;;
app)
echo "Starting web app (using development server)..."
flask run -p $PORT --with-threads --reload --debugger --host=0.0.0.0
flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
;;
app-gunicorn)
echo "Starting web app..."

View File

@@ -23,4 +23,4 @@
export SERVER_THREADS_AMOUNT=8
# start up the web server
/app/docker/entrypoints/run-server.sh
/usr/bin/run-server.sh

View File

@@ -1,28 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ------------------------------------------------------------------------
# Creates the examples database and respective user. This database location
# and access credentials are defined on the environment variables
# ------------------------------------------------------------------------
set -e
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" <<-EOSQL
CREATE DATABASE superset_cypress;
EOSQL

View File

@@ -35,10 +35,8 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
echo "Running `npm install`"
npm install
echo "Start webpack dev server"
# start the webpack dev server, serving dynamically at http://localhost:9000
# it proxies to the backend served at http://localhost:8088
npm run dev-server
echo "Running frontend"
npm run dev
else
echo "Skipping frontend build steps - YOU NEED TO RUN IT MANUALLY ON THE HOST!"

View File

@@ -30,18 +30,24 @@ fi
echo_step() {
cat <<EOF
######################################################################
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
######################################################################
EOF
}
ADMIN_PASSWORD="${ADMIN_PASSWORD:-admin}"
# If Cypress run overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then
ADMIN_PASSWORD="general"
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export POSTGRES_DB=superset_cypress
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset_cypress
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
fi
# Initialize the database
echo_step "1" "Starting" "Applying DB migrations"
@@ -50,16 +56,12 @@ echo_step "1" "Complete" "Applying DB migrations"
# Create an admin user
echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
if [ "$CYPRESS_CONFIG" == "true" ]; then
superset load_test_users
else
superset fab create-admin \
--username admin \
--email admin@superset.com \
--password "$ADMIN_PASSWORD" \
--firstname Superset \
--lastname Admin
fi
superset fab create-admin \
--username admin \
--firstname Superset \
--lastname Admin \
--email admin@superset.com \
--password "$ADMIN_PASSWORD"
echo_step "2" "Complete" "Setting up admin user"
# Create default roles and permissions
echo_step "3" "Starting" "Setting up roles and perms"
@@ -71,9 +73,10 @@ if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
echo_step "4" "Starting" "Loading examples"
# If Cypress run which consumes superset_test_config load required data for tests
if [ "$CYPRESS_CONFIG" == "true" ]; then
superset load_test_users
superset load_examples --load-test-data
else
superset load_examples
superset load_examples --force
fi
echo_step "4" "Complete" "Loading examples"
fi

View File

@@ -112,12 +112,6 @@ http {
proxy_set_header Host $host;
}
location /static {
proxy_pass http://host.docker.internal:9000; # Proxy to superset-node
proxy_http_version 1.1;
proxy_set_header Host $host;
}
location / {
proxy_pass http://superset_app;
proxy_set_header Host $host;

View File

@@ -47,10 +47,10 @@ fi
# Choose whether to use pip cache
if $USE_CACHE; then
echo "Using pip cache..."
uv pip install "${ARGS[@]}"
uv pip install --system "${ARGS[@]}"
else
echo "Disabling pip cache..."
uv pip install --no-cache-dir "${ARGS[@]}"
uv pip install --system --no-cache-dir "${ARGS[@]}"
fi
# Remove build-essential if it was installed

View File

@@ -22,7 +22,6 @@
#
import logging
import os
import sys
from celery.schedules import crontab
from flask_caching.backends.filesystemcache import FileSystemCache
@@ -100,26 +99,11 @@ CELERY_CONFIG = CeleryConfig
FEATURE_FLAGS = {"ALERT_REPORTS": True}
ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
WEBDRIVER_BASEURL = "http://superset:8088/" # When using docker compose baseurl should be http://superset_app:8088/ # noqa: E501
WEBDRIVER_BASEURL = "http://superset:8088/" # When using docker compose baseurl should be http://superset_app:8088/
# The base URL for the email report hyperlinks.
WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
SQLLAB_CTAS_NO_LIMIT = True
log_level_text = os.getenv("SUPERSET_LOG_LEVEL", "INFO")
LOG_LEVEL = getattr(logging, log_level_text.upper(), logging.INFO)
if os.getenv("CYPRESS_CONFIG") == "true":
# When running the service as a cypress backend, we need to import the config
# located @ tests/integration_tests/superset_test_config.py
base_dir = os.path.dirname(__file__)
module_folder = os.path.abspath(
os.path.join(base_dir, "../../tests/integration_tests/")
)
sys.path.insert(0, module_folder)
from superset_test_config import * # noqa
sys.path.pop(0)
#
# Optionally import superset_config_docker.py (which will have been included on
# the PYTHONPATH) in order to allow for local settings to be overridden
@@ -129,7 +113,7 @@ try:
from superset_config_docker import * # noqa
logger.info(
f"Loaded your Docker configuration at [{superset_config_docker.__file__}]"
f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"
)
except ImportError:
logger.info("Using default Docker config...")

View File

@@ -1,47 +0,0 @@
/* eslint-env node */
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
module.exports = {
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'plugin:react/recommended',
'plugin:prettier/recommended',
],
parser: '@typescript-eslint/parser',
parserOptions: {
ecmaFeatures: {
jsx: true,
},
ecmaVersion: 2020,
sourceType: 'module',
},
plugins: ['@typescript-eslint', 'react', 'prettier'],
rules: {
'react/react-in-jsx-scope': 'off',
'react/prop-types': 'off',
'@typescript-eslint/explicit-module-boundary-types': 'off',
},
settings: {
react: {
version: 'detect',
},
},
ignorePatterns: ['build/**/*', '.docusaurus/**/*', 'node_modules/**/*'],
};

View File

@@ -1 +1 @@
v20.18.3
v20.16.0

View File

@@ -18,6 +18,6 @@ under the License.
-->
This is the public documentation site for Superset, built using
[Docusaurus 3](https://docusaurus.io/). See
[Docusaurus 2](https://docusaurus.io/). See
[CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on
contributing to documentation.

View File

@@ -1,4 +1,3 @@
/* eslint-env node */
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file

View File

@@ -63,7 +63,6 @@
"Fiji",
"Finland",
"France",
"France (with overseas)",
"France (regions)",
"French Polynesia",
"Gabon",

View File

@@ -4,6 +4,7 @@ hide_title: true
sidebar_position: 10
---
import { Buffer } from 'buffer/index.js';
import SwaggerUI from 'swagger-ui-react';
import openapi from '/resources/openapi.json';
import 'swagger-ui-react/swagger-ui.css';

View File

@@ -25,9 +25,6 @@ Alerts and reports are disabled by default. To turn them on, you need to do some
- At least one of those must be configured, depending on what you want to use:
- emails: `SMTP_*` settings
- Slack messages: `SLACK_API_TOKEN`
- Users can customize the email subject by including date code placeholders, which will automatically be replaced with the corresponding UTC date when the email is sent. To enable this functionality, activate the `"DATE_FORMAT_IN_EMAIL_SUBJECT"` [feature flag](/docs/configuration/configuring-superset#feature-flags). This enables date formatting in email subjects, preventing all reporting emails from being grouped into the same thread (optional for the reporting feature).
- Use date codes from [strftime.org](https://strftime.org/) to create the email subject.
- If no date code is provided, the original string will be used as the email subject.
##### Disable dry-run mode
@@ -92,7 +89,6 @@ You can find documentation about each field in the default `config.py` in the Gi
You need to replace default values with your custom Redis, Slack and/or SMTP config.
Superset uses Celery beat and Celery worker(s) to send alerts and reports.
- The beat is the scheduler that tells the worker when to perform its tasks. This schedule is defined when you create the alert or report.
- The worker will process the tasks that need to be performed when an alert or report is fired.
@@ -144,7 +140,7 @@ SLACK_API_TOKEN = "xoxb-"
SMTP_HOST = "smtp.sendgrid.net" # change to your host
SMTP_PORT = 2525 # your port, e.g. 587
SMTP_STARTTLS = True
SMTP_SSL_SERVER_AUTH = True # If you're using an SMTP server with a valid certificate
SMTP_SSL_SERVER_AUTH = True # If your using an SMTP server with a valid certificate
SMTP_SSL = False
SMTP_USER = "your_user" # use the empty string "" if using an unauthenticated SMTP server
SMTP_PASSWORD = "your_password" # use the empty string "" if using an unauthenticated SMTP server
@@ -181,13 +177,15 @@ By default, Alerts and Reports are executed as the owner of the alert/report obj
just change the config as follows (`admin` in this example):
```python
from superset.tasks.types import FixedExecutor
from superset.tasks.types import ExecutorType
ALERT_REPORTS_EXECUTORS = [FixedExecutor("admin")]
THUMBNAIL_SELENIUM_USER = 'admin'
ALERT_REPORTS_EXECUTE_AS = [ExecutorType.SELENIUM]
```
Please refer to `ExecutorType` in the codebase for other executor types.
**Important notes**
- Be mindful of the concurrency setting for celery (using `-c 4`). Selenium/webdriver instances can
@@ -199,6 +197,7 @@ Please refer to `ExecutorType` in the codebase for other executor types.
- Adjust `WEBDRIVER_BASEURL` in your configuration file if celery workers cant access Superset via
its default value of `http://0.0.0.0:8080/`.
It's also possible to specify a minimum interval between each report's execution through the config file:
``` python
@@ -304,7 +303,6 @@ One symptom of an invalid connection to an email server is receiving an error of
Confirm via testing that your outbound email configuration is correct. Here is the simplest test, for an un-authenticated email SMTP email service running on port 25. If you are sending over SSL, for instance, study how [Superset's codebase sends emails](https://github.com/apache/superset/blob/master/superset/utils/core.py#L818) and then test with those commands and arguments.
Start Python in your worker environment, replace all example values, and run:
```python
import smtplib
from email.mime.multipart import MIMEMultipart
@@ -326,7 +324,6 @@ mailserver.quit()
This should send an email.
Possible fixes:
- Some cloud hosts disable outgoing unauthenticated SMTP email to prevent spam. For instance, [Azure blocks port 25 by default on some machines](https://learn.microsoft.com/en-us/azure/virtual-network/troubleshoot-outbound-smtp-connectivity). Enable that port or use another sending method.
- Use another set of SMTP credentials that you verify works in this setup.

View File

@@ -42,13 +42,13 @@ CELERY_CONFIG = CeleryConfig
To start a Celery worker to leverage the configuration, run the following command:
```bash
```
celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4
```
To start a job which schedules periodic background jobs, run the following command:
```bash
```
celery --app=superset.tasks.celery_app:app beat
```
@@ -93,12 +93,12 @@ issues arise. Please clear your existing results cache store when upgrading an e
Flower is a web based tool for monitoring the Celery cluster which you can install from pip:
```bash
```python
pip install flower
```
You can run flower using:
```bash
```
celery --app=superset.tasks.celery_app:app flower
```

View File

@@ -17,7 +17,6 @@ Caching can be configured by providing dictionaries in
`superset_config.py` that comply with [the Flask-Caching config specifications](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching).
The following cache configurations can be customized in this way:
- Dashboard filter state (required): `FILTER_STATE_CACHE_CONFIG`.
- Explore chart form data (required): `EXPLORE_FORM_DATA_CACHE_CONFIG`
- Metadata cache (optional): `CACHE_CONFIG`
@@ -82,7 +81,7 @@ See [Async Queries via Celery](/docs/configuration/async-queries-celery) for det
## Caching Thumbnails
This is an optional feature that can be turned on by activating its [feature flag](/docs/configuration/configuring-superset#feature-flags) on config:
This is an optional feature that can be turned on by activating its [feature flag](/docs/configuration/configuring-superset#feature-flags) on config:
```
FEATURE_FLAGS = {
@@ -95,11 +94,13 @@ By default thumbnails are rendered per user, and will fall back to the Selenium
To always render thumbnails as a fixed user (`admin` in this example), use the following configuration:
```python
from superset.tasks.types import FixedExecutor
from superset.tasks.types import ExecutorType
THUMBNAIL_EXECUTORS = [FixedExecutor("admin")]
THUMBNAIL_SELENIUM_USER = "admin"
THUMBNAIL_EXECUTE_AS = [ExecutorType.SELENIUM]
```
For this feature you will need a cache system and celery workers. All thumbnails are stored on cache
and are processed asynchronously by the workers.
@@ -129,6 +130,8 @@ def init_thumbnail_cache(app: Flask) -> S3Cache:
THUMBNAIL_CACHE_CONFIG = init_thumbnail_cache
# Async selenium thumbnail task will use the following user
THUMBNAIL_SELENIUM_USER = "Admin"
```
Using the above example cache keys for dashboards will be `superset_thumb__dashboard__{ID}`. You can

View File

@@ -117,7 +117,7 @@ Your deployment must use a complex, unique key.
### Rotating to a newer SECRET_KEY
If you wish to change your existing SECRET_KEY, add the existing SECRET_KEY to your `superset_config.py` file as
`PREVIOUS_SECRET_KEY =`and provide your new key as `SECRET_KEY =`. You can find your current SECRET_KEY with these
`PREVIOUS_SECRET_KEY = `and provide your new key as `SECRET_KEY =`. You can find your current SECRET_KEY with these
commands - if running Superset with Docker, execute from within the Superset application container:
```python
@@ -141,10 +141,10 @@ database engine on a separate host or container.
Superset supports the following database engines/versions:
| Database Engine | Supported Versions |
| ----------------------------------------- | ---------------------------------------- |
| [PostgreSQL](https://www.postgresql.org/) | 10.X, 11.X, 12.X, 13.X, 14.X, 15.X, 16.X |
| [MySQL](https://www.mysql.com/) | 5.7, 8.X |
| Database Engine | Supported Versions |
| ----------------------------------------- | ---------------------------------- |
| [PostgreSQL](https://www.postgresql.org/) | 10.X, 11.X, 12.X, 13.X, 14.X, 15.X |
| [MySQL](https://www.mysql.com/) | 5.7, 8.X |
Use the following database drivers and connection strings:
@@ -283,7 +283,7 @@ class CustomSsoSecurityManager(SupersetSecurityManager):
...
```
This file must be located in the same directory as `superset_config.py` with the name
This file must be located at the same directory than `superset_config.py` with the name
`custom_sso_security_manager.py`. Finally, add the following 2 lines to `superset_config.py`:
```
@@ -300,7 +300,6 @@ CUSTOM_SECURITY_MANAGER = CustomSsoSecurityManager
- If an OAuth2 authorization server supports OpenID Connect 1.0, you could configure its configuration
document URL only without providing `api_base_url`, `access_token_url`, `authorize_url` and other
required options like user info endpoint, jwks uri etc. For instance:
```python
OAUTH_PROVIDERS = [
{ 'name':'egaSSO',
@@ -314,15 +313,12 @@ CUSTOM_SECURITY_MANAGER = CustomSsoSecurityManager
}
]
```
### Keycloak-Specific Configuration using Flask-OIDC
If you are using Keycloak as OpenID Connect 1.0 Provider, the above configuration based on [`Authlib`](https://authlib.org/) might not work. In this case using [`Flask-OIDC`](https://pypi.org/project/flask-oidc/) is a viable option.
Make sure the pip package [`Flask-OIDC`](https://pypi.org/project/flask-oidc/) is installed on the webserver. This was successfully tested using version 2.2.0. This package requires [`Flask-OpenID`](https://pypi.org/project/Flask-OpenID/) as a dependency.
Make sure the pip package [`Flask-OIDC`](https://pypi.org/project/flask-oidc/) is installed on the webserver. This was succesfully tested using version 2.2.0. This package requires [`Flask-OpenID`](https://pypi.org/project/Flask-OpenID/) as a dependency.
The following code defines a new security manager. Add it to a new file named `keycloak_security_manager.py`, placed in the same directory as your `superset_config.py` file.
```python
from flask_appbuilder.security.manager import AUTH_OID
from superset.security import SupersetSecurityManager
@@ -377,9 +373,7 @@ class AuthOIDCView(AuthOIDView):
return redirect(
oidc.client_secrets.get('issuer') + '/protocol/openid-connect/logout?redirect_uri=' + quote(redirect_url))
```
Then add to your `superset_config.py` file:
```python
from keycloak_security_manager import OIDCSecurityManager
from flask_appbuilder.security.manager import AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH
@@ -399,9 +393,7 @@ AUTH_USER_REGISTRATION = True
# The default user self registration role
AUTH_USER_REGISTRATION_ROLE = 'Public'
```
Store your client-specific OpenID information in a file called `client_secret.json`. Create this file in the same directory as `superset_config.py`:
```json
{
"<myOpenIDProvider>": {
@@ -418,7 +410,6 @@ Store your client-specific OpenID information in a file called `client_secret.js
}
}
```
## LDAP Authentication
FAB supports authenticating user credentials against an LDAP server.
@@ -441,7 +432,6 @@ AUTH_ROLES_MAPPING = {
"superset_admins": ["Admin"],
}
```
### Mapping LDAP groups to Superset roles
The following `AUTH_ROLES_MAPPING` dictionary would map the LDAP DN "cn=superset_users,ou=groups,dc=example,dc=com" to the Superset roles "Gamma" as well as "Alpha", and the LDAP DN "cn=superset_admins,ou=groups,dc=example,dc=com" to the Superset role "Admin".
@@ -452,7 +442,6 @@ AUTH_ROLES_MAPPING = {
"cn=superset_admins,ou=groups,dc=example,dc=com": ["Admin"],
}
```
Note: This requires `AUTH_LDAP_SEARCH` to be set. For more details, please see the [FAB Security documentation](https://flask-appbuilder.readthedocs.io/en/latest/security.html).
### Syncing roles at login
@@ -486,7 +475,7 @@ def FLASK_APP_MUTATOR(app: Flask) -> None:
To support a diverse set of users, Superset has some features that are not enabled by default. For
example, some users have stronger security restrictions, while some others may not. So Superset
allows users to enable or disable some features by config. For feature owners, you can add optional
allow users to enable or disable some features by config. For feature owners, you can add optional
functionalities in Superset, but will be only affected by a subset of users.
You can enable or disable features with flag from `superset_config.py`:

View File

@@ -31,17 +31,18 @@ install new database drivers into your Superset configuration.
### Supported Databases and Dependencies
Some of the recommended packages are shown below. Please refer to
[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) for the versions that
are compatible with Superset.
| <div style={{width: '150px'}}>Database</div> | PyPI package | Connection String |
| --------------------------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
| [AWS Athena](/docs/configuration/databases#aws-athena) | `pip install pyathena[pandas]` , `pip install PyAthenaJDBC` | `awsathena+rest://{access_key_id}:{access_key}@athena.{region}.amazonaws.com/{schema}?s3_staging_dir={s3_staging_dir}&...` |
| [AWS Athena](/docs/configuration/databases#aws-athena) | `pip install pyathena[pandas]` , `pip install PyAthenaJDBC` | `awsathena+rest://{access_key_id}:{access_key}@athena.{region}.amazonaws.com/{schema}?s3_staging_dir={s3_staging_dir}&... ` |
| [AWS DynamoDB](/docs/configuration/databases#aws-dynamodb) | `pip install pydynamodb` | `dynamodb://{access_key_id}:{secret_access_key}@dynamodb.{region_name}.amazonaws.com?connector=superset` |
| [AWS Redshift](/docs/configuration/databases#aws-redshift) | `pip install sqlalchemy-redshift` | `redshift+psycopg2://<userName>:<DBPassword>@<AWS End Point>:5439/<Database Name>` |
| [AWS Redshift](/docs/configuration/databases#aws-redshift) | `pip install sqlalchemy-redshift` | ` redshift+psycopg2://<userName>:<DBPassword>@<AWS End Point>:5439/<Database Name>` |
| [Apache Doris](/docs/configuration/databases#apache-doris) | `pip install pydoris` | `doris://<User>:<Password>@<Host>:<Port>/<Catalog>.<Database>` |
| [Apache Drill](/docs/configuration/databases#apache-drill) | `pip install sqlalchemy-drill` | `drill+sadrill://<username>:<password>@<host>:<port>/<storage_plugin>`, often useful: `?use_ssl=True/False` |
| [Apache Drill](/docs/configuration/databases#apache-drill) | `pip install sqlalchemy-drill` | `drill+sadrill:// For JDBC drill+jdbc://` |
| [Apache Druid](/docs/configuration/databases#apache-druid) | `pip install pydruid` | `druid://<User>:<password>@<Host>:<Port-default-9088>/druid/v2/sql` |
| [Apache Hive](/docs/configuration/databases#hive) | `pip install pyhive` | `hive://hive@{hostname}:{port}/{database}` |
| [Apache Impala](/docs/configuration/databases#apache-impala) | `pip install impyla` | `impala://{hostname}:{port}/{database}` |
@@ -67,24 +68,21 @@ are compatible with Superset.
| [IBM Netezza Performance Server](/docs/configuration/databases#ibm-netezza-performance-server) | `pip install nzalchemy` | `netezza+nzpy://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
| [MySQL](/docs/configuration/databases#mysql) | `pip install mysqlclient` | `mysql://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
| [OceanBase](/docs/configuration/databases#oceanbase) | `pip install oceanbase_py` | `oceanbase://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
| [Oracle](/docs/configuration/databases#oracle) | `pip install cx_Oracle` | `oracle://<username>:<password>@<hostname>:<port>` |
| [Parseable](/docs/configuration/databases#parseable) | `pip install sqlalchemy-parseable` | `parseable://<UserName>:<DBPassword>@<Database Host>/<Stream Name>` |
| [Oracle](/docs/configuration/databases#oracle) | `pip install cx_Oracle` | `oracle://` |
| [PostgreSQL](/docs/configuration/databases#postgres) | `pip install psycopg2` | `postgresql://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
| [Presto](/docs/configuration/databases#presto) | `pip install pyhive` | `presto://{username}:{password}@{hostname}:{port}/{database}` |
| [Presto](/docs/configuration/databases#presto) | `pip install pyhive` | `presto://` |
| [Rockset](/docs/configuration/databases#rockset) | `pip install rockset-sqlalchemy` | `rockset://<api_key>:@<api_server>` |
| [SAP Hana](/docs/configuration/databases#hana) | `pip install hdbcli sqlalchemy-hana` or `pip install apache-superset[hana]` | `hana://{username}:{password}@{host}:{port}` |
| [StarRocks](/docs/configuration/databases#starrocks) | `pip install starrocks` | `starrocks://<User>:<Password>@<Host>:<Port>/<Catalog>.<Database>` |
| [Snowflake](/docs/configuration/databases#snowflake) | `pip install snowflake-sqlalchemy` | `snowflake://{user}:{password}@{account}.{region}/{database}?role={role}&warehouse={warehouse}` |
| SQLite | No additional library needed | `sqlite://path/to/file.db?check_same_thread=false` |
| [SQL Server](/docs/configuration/databases#sql-server) | `pip install pymssql` | `mssql+pymssql://<Username>:<Password>@<Host>:<Port-default:1433>/<Database Name>` |
| [TDengine](/docs/configuration/databases#tdengine) | `pip install taospy` `pip install taos-ws-py` | `taosws://<user>:<password>@<host>:<port>` |
| [SQL Server](/docs/configuration/databases#sql-server) | `pip install pymssql` | `mssql+pymssql://` |
| [Teradata](/docs/configuration/databases#teradata) | `pip install teradatasqlalchemy` | `teradatasql://{user}:{password}@{host}` |
| [TimescaleDB](/docs/configuration/databases#timescaledb) | `pip install psycopg2` | `postgresql://<UserName>:<DBPassword>@<Database Host>:<Port>/<Database Name>` |
| [Trino](/docs/configuration/databases#trino) | `pip install trino` | `trino://{username}:{password}@{hostname}:{port}/{catalog}` |
| [Vertica](/docs/configuration/databases#vertica) | `pip install sqlalchemy-vertica-python` | `vertica+vertica_python://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
| [YDB](/docs/configuration/databases#ydb) | `pip install ydb-sqlalchemy` | `ydb://{host}:{port}/{database_name}` |
| [YugabyteDB](/docs/configuration/databases#yugabytedb) | `pip install psycopg2` | `postgresql://<UserName>:<DBPassword>@<Database Host>/<Database Name>` |
---
Note that many other databases are supported, the main criteria being the existence of a functional
@@ -185,6 +183,7 @@ purposes of isolating the problem.
Repeat this process for each type of database you want Superset to connect to.
### Database-specific Instructions
#### Ascend.io
@@ -210,12 +209,14 @@ You'll need the following setting values to form the connection string:
- **Catalog**: Catalog Name
- **Database**: Database Name
Here's what the connection string looks like:
```
doris://<User>:<Password>@<Host>:<Port>/<Catalog>.<Database>
```
#### AWS Athena
##### PyAthenaJDBC
@@ -245,7 +246,6 @@ awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name
```
The PyAthena library also allows to assume a specific IAM role which you can define by adding following parameters in Superset's Athena database connection UI under ADVANCED --> Other --> ENGINE PARAMETERS.
```json
{
"connect_args": {
@@ -268,6 +268,7 @@ dynamodb://{aws_access_key_id}:{aws_secret_access_key}@dynamodb.{region_name}.am
To get more documentation, please visit: [PyDynamoDB WIKI](https://github.com/passren/PyDynamoDB/wiki/5.-Superset).
#### AWS Redshift
The [sqlalchemy-redshift](https://pypi.org/project/sqlalchemy-redshift/) library is the recommended
@@ -283,6 +284,7 @@ You'll need to set the following values to form the connection string:
- **Database Name**: Database Name
- **Port**: default 5439
##### psycopg2
Here's what the SQLALCHEMY URI looks like:
@@ -291,6 +293,7 @@ Here's what the SQLALCHEMY URI looks like:
redshift+psycopg2://<userName>:<DBPassword>@<AWS End Point>:5439/<Database Name>
```
##### redshift_connector
Here's what the SQLALCHEMY URI looks like:
@@ -299,7 +302,8 @@ Here's what the SQLALCHEMY URI looks like:
redshift+redshift_connector://<userName>:<DBPassword>@<AWS End Point>:5439/<Database Name>
```
###### Using IAM-based credentials with Redshift cluster
###### Using IAM-based credentials with Redshift cluster:
[Amazon redshift cluster](https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-clusters.html) also supports generating temporary IAM-based database user credentials.
@@ -310,10 +314,10 @@ You have to define the following arguments in Superset's redshift database conne
```
{"connect_args":{"iam":true,"database":"<database>","cluster_identifier":"<cluster_identifier>","db_user":"<db_user>"}}
```
and SQLALCHEMY URI should be set to `redshift+redshift_connector://`
###### Using IAM-based credentials with Redshift serverless
###### Using IAM-based credentials with Redshift serverless:
[Redshift serverless](https://docs.aws.amazon.com/redshift/latest/mgmt/serverless-whatis.html) supports connection using IAM roles.
@@ -325,6 +329,8 @@ You have to define the following arguments in Superset's redshift database conne
{"connect_args":{"iam":true,"is_serverless":true,"serverless_acct_id":"<aws account number>","serverless_work_group":"<redshift work group>","database":"<database>","user":"IAMR:<superset iam role name>"}}
```
#### ClickHouse
To use ClickHouse with Superset, you will need to install the `clickhouse-connect` Python library:
@@ -357,6 +363,8 @@ uses the default user without a password (and doesn't encrypt the connection):
clickhousedb://localhost/default
```
#### CockroachDB
The recommended connector library for CockroachDB is
@@ -368,12 +376,13 @@ The expected connection string is formatted as follows:
cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable
```
#### Couchbase
The Couchbase's Superset connection is designed to support two services: Couchbase Analytics and Couchbase Columnar.
The recommended connector library for couchbase is
[couchbase-sqlalchemy](https://github.com/couchbase/couchbase-sqlalchemy).
```
pip install couchbase-sqlalchemy
```
@@ -384,25 +393,22 @@ The expected connection string is formatted as follows:
couchbase://{username}:{password}@{hostname}:{port}?truststorepath={certificate path}?ssl={true/false}
```
#### CrateDB
The connector library for CrateDB is [sqlalchemy-cratedb].
We recommend to add the following item to your `requirements.txt` file:
```
sqlalchemy-cratedb>=0.40.1,<1
```
An SQLAlchemy connection string for [CrateDB Self-Managed] on localhost,
for evaluation purposes, looks like this:
```
crate://crate@127.0.0.1:4200
```
An SQLAlchemy connection string for connecting to [CrateDB Cloud] looks like
this:
```
crate://<username>:<password>@<clustername>.cratedb.net:4200/?ssl=true
```
@@ -410,7 +416,6 @@ crate://<username>:<password>@<clustername>.cratedb.net:4200/?ssl=true
Follow the steps [here](/docs/configuration/databases#installing-database-drivers)
to install the CrateDB connector package when setting up Superset locally using
Docker Compose.
```
echo "sqlalchemy-cratedb" >> ./docker/requirements-local.txt
```
@@ -419,6 +424,7 @@ echo "sqlalchemy-cratedb" >> ./docker/requirements-local.txt
[CrateDB Self-Managed]: https://cratedb.com/product/self-managed
[sqlalchemy-cratedb]: https://pypi.org/project/sqlalchemy-cratedb/
#### Databend
The recommended connector library for Databend is [databend-sqlalchemy](https://pypi.org/project/databend-sqlalchemy/).
@@ -436,6 +442,7 @@ Here's a connection string example of Superset connecting to a Databend database
databend://user:password@localhost:8000/default?secure=false
```
#### Databricks
Databricks now offer a native DB API 2.0 driver, `databricks-sql-connector`, that can be used with the `sqlalchemy-databricks` dialect. You can install both with:
@@ -519,6 +526,7 @@ For a connection to a SQL endpoint you need to use the HTTP path from the endpoi
{"connect_args": {"http_path": "/sql/1.0/endpoints/****", "driver_path": "/path/to/odbc/driver"}}
```
#### Denodo
The recommended connector library for Denodo is
@@ -530,6 +538,7 @@ The expected connection string is formatted as follows (default port is 9996):
denodo://{username}:{password}@{hostname}:{port}/{database}
```
#### Dremio
The recommended connector library for Dremio is
@@ -550,6 +559,7 @@ dremio+flight://{username}:{password}@{host}:{port}/dremio
This [blog post by Dremio](https://www.dremio.com/tutorials/dremio-apache-superset/) has some
additional helpful instructions on connecting Superset to Dremio.
#### Apache Drill
##### SQLAlchemy
@@ -591,6 +601,8 @@ We recommend reading the
the [GitHub README](https://github.com/JohnOmernik/sqlalchemy-drill#usage-with-odbc) to learn how to
work with Drill through ODBC.
import useBaseUrl from "@docusaurus/useBaseUrl";
#### Apache Druid
@@ -604,7 +616,6 @@ The connection string looks like:
```
druid://<User>:<password>@<Host>:<Port-default-9088>/druid/v2/sql
```
Here's a breakdown of the key components of this connection string:
- `User`: username portion of the credentials needed to connect to your database
@@ -633,7 +644,7 @@ To disable SSL verification, add the following to the **Extras** field:
```
engine_params:
{"connect_args":
{"scheme": "https", "ssl_verify_cert": false}}
{"scheme": "https", "ssl_verify_cert": false}}
```
##### Aggregations
@@ -657,6 +668,7 @@ much like you would create an aggregation manually, but specify `postagg` as a `
then have to provide a valid json post-aggregation definition (as specified in the Druid docs) in
the JSON field.
#### Elasticsearch
The recommended connector library for Elasticsearch is
@@ -705,7 +717,7 @@ Then register your table with the alias name logstash_all
By default, Superset uses UTC time zone for elasticsearch query. If you need to specify a time zone,
please edit your Database and enter the settings of your specified time zone in the Other > ENGINE PARAMETERS:
```json
```
{
"connect_args": {
"time_zone": "Asia/Shanghai"
@@ -727,6 +739,8 @@ To disable SSL verification, add the following to the **SQLALCHEMY URI** field:
elasticsearch+https://{user}:{password}@{host}:9200/?verify_certs=False
```
#### Exasol
The recommended connector library for Exasol is
@@ -738,6 +752,7 @@ The connection string for Exasol looks like this:
exa+pyodbc://{username}:{password}@{hostname}:{port}/my_schema?CONNECTIONLCALL=en_US.UTF-8&driver=EXAODBC
```
#### Firebird
The recommended connector library for Firebird is [sqlalchemy-firebird](https://pypi.org/project/sqlalchemy-firebird/).
@@ -755,6 +770,7 @@ Here's a connection string example of Superset connecting to a local Firebird da
firebird+fdb://SYSDBA:masterkey@192.168.86.38:3050//Library/Frameworks/Firebird.framework/Versions/A/Resources/examples/empbuild/employee.fdb
```
#### Firebolt
The recommended connector library for Firebolt is [firebolt-sqlalchemy](https://pypi.org/project/firebolt-sqlalchemy/).
@@ -785,7 +801,7 @@ The recommended connector library for BigQuery is
Follow the steps [here](/docs/configuration/databases#installing-drivers-in-docker-images) about how to
install new database drivers when setting up Superset locally via docker compose.
```bash
```
echo "sqlalchemy-bigquery" >> ./docker/requirements-local.txt
```
@@ -798,7 +814,7 @@ credentials file (as a JSON).
appropriate BigQuery datasets, and download the JSON configuration file for the service account.
2. In Superset, you can either upload that JSON or add the JSON blob in the following format (this should be the content of your credential JSON file):
```json
```
{
"type": "service_account",
"project_id": "...",
@@ -826,7 +842,7 @@ credentials file (as a JSON).
Go to the **Advanced** tab, Add a JSON blob to the **Secure Extra** field in the database configuration form with
the following format:
```json
```
{
"credentials_info": <contents of credentials JSON file>
}
@@ -834,7 +850,7 @@ credentials file (as a JSON).
The resulting file should have this structure:
```json
```
{
"credentials_info": {
"type": "service_account",
@@ -861,6 +877,8 @@ To be able to upload CSV or Excel files to BigQuery in Superset, you'll need to
Currently, the Google BigQuery Python SDK is not compatible with `gevent`, due to some dynamic monkeypatching on python core library by `gevent`.
So, when you deploy Superset with `gunicorn` server, you have to use worker type except `gevent`.
#### Google Sheets
Google Sheets has a very limited
@@ -871,6 +889,7 @@ There are a few steps involved in connecting Superset to Google Sheets. This
[tutorial](https://preset.io/blog/2020-06-01-connect-superset-google-sheets/) has the most up to date
instructions on setting up this connection.
#### Hana
The recommended connector library is [sqlalchemy-hana](https://github.com/SAP/sqlalchemy-hana).
@@ -881,6 +900,7 @@ The connection string is formatted as follows:
hana://{username}:{password}@{host}:{port}
```
#### Apache Hive
The [pyhive](https://pypi.org/project/PyHive/) library is the recommended way to connect to Hive through SQLAlchemy.
@@ -891,6 +911,7 @@ The expected connection string is formatted as follows:
hive://hive@{hostname}:{port}/{database}
```
#### Hologres
Hologres is a real-time interactive analytics service developed by Alibaba Cloud. It is fully compatible with PostgreSQL 11 and integrates seamlessly with the big data ecosystem.
@@ -909,6 +930,7 @@ The connection string looks like:
postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}
```
#### IBM DB2
The [IBM_DB_SA](https://github.com/ibmdb/python-ibmdbsa/tree/master/ibm_db_sa) library provides a
@@ -926,6 +948,7 @@ There are two DB2 dialect versions implemented in SQLAlchemy. If you are connect
ibm_db_sa://{username}:{passport}@{hostname}:{port}/{database}
```
#### Apache Impala
The recommended connector library to Apache Impala is [impyla](https://github.com/cloudera/impyla).
@@ -936,6 +959,7 @@ The expected connection string is formatted as follows:
impala://{hostname}:{port}/{database}
```
#### Kusto
The recommended connector library for Kusto is
@@ -956,6 +980,7 @@ kustokql+https://{cluster_url}/{database}?azure_ad_client_id={azure_ad_client_id
Make sure the user has privileges to access and use all required
databases/tables/views.
#### Apache Kylin
The recommended connector library for Apache Kylin is
@@ -967,6 +992,10 @@ The expected connection string is formatted as follows:
kylin://<username>:<password>@<hostname>:<port>/<project>?<param1>=<value1>&<param2>=<value2>
```
#### MySQL
The recommended connector library for MySQL is [mysqlclient](https://pypi.org/project/mysqlclient/).
@@ -991,6 +1020,7 @@ One problem with `mysqlclient` is that it will fail to connect to newer MySQL da
mysql+mysqlconnector://{username}:{password}@{host}/{database}
```
#### IBM Netezza Performance Server
The [nzalchemy](https://pypi.org/project/nzalchemy/) library provides a
@@ -1007,19 +1037,21 @@ netezza+nzpy://{username}:{password}@{hostname}:{port}/{database}
The [sqlalchemy-oceanbase](https://pypi.org/project/oceanbase_py/) library is the recommended
way to connect to OceanBase through SQLAlchemy.
The connection string for OceanBase looks like this:
```
oceanbase://<User>:<Password>@<Host>:<Port>/<Database>
```
#### Ocient DB
The recommended connector library for Ocient is [sqlalchemy-ocient](https://pypi.org/project/sqlalchemy-ocient).
##### Install the Ocient Driver
```bash
```
pip install sqlalchemy-ocient
```
@@ -1042,25 +1074,8 @@ The connection string is formatted as follows:
oracle://<username>:<password>@<hostname>:<port>
```
#### Parseable
[Parseable](https://www.parseable.io) is a distributed log analytics database that provides SQL-like query interface for log data. The recommended connector library is [sqlalchemy-parseable](https://github.com/parseablehq/sqlalchemy-parseable).
The connection string is formatted as follows:
```
parseable://<username>:<password>@<hostname>:<port>/<stream_name>
```
For example:
```
parseable://admin:admin@demo.parseable.com:443/ingress-nginx
```
Note: The stream_name in the URI represents the Parseable logstream you want to query. You can use both HTTP (port 80) and HTTPS (port 443) connections.
>>>>>>>
#### Apache Pinot
The recommended connector library for Apache Pinot is [pinotdb](https://pypi.org/project/pinotdb/).
@@ -1079,8 +1094,7 @@ pinot://<username>:<password>@<pinot-broker-host>:<pinot-broker-port>/query/sql?
If you want to use explore view or joins, window functions, etc. then enable [multi-stage query engine](https://docs.pinot.apache.org/reference/multi-stage-engine).
Add below argument while creating database connection in Advanced -> Other -> ENGINE PARAMETERS
```json
```
{"connect_args":{"use_multistage_engine":"true"}}
```
@@ -1120,6 +1134,7 @@ More information about PostgreSQL connection options can be found in the
and the
[PostgreSQL docs](https://www.postgresql.org/docs/9.1/libpq-connect.html#LIBPQ-PQCONNECTDBPARAMS).
#### Presto
The [pyhive](https://pypi.org/project/PyHive/) library is the recommended way to connect to Presto through SQLAlchemy.
@@ -1145,7 +1160,7 @@ presto://datascientist:securepassword@presto.example.com:8080/hive
By default Superset assumes the most recent version of Presto is being used when querying the
datasource. If youre using an older version of Presto, you can configure it in the extra parameter:
```json
```
{
"version": "0.123"
}
@@ -1153,7 +1168,7 @@ datasource. If youre using an older version of Presto, you can configure it i
SSL Secure extra add json config to extra connection information.
```json
```
{
"connect_args":
{"protocol": "https",
@@ -1162,6 +1177,8 @@ SSL Secure extra add json config to extra connection information.
}
```
#### RisingWave
The recommended connector library for RisingWave is
@@ -1173,6 +1190,7 @@ The expected connection string is formatted as follows:
risingwave://root@{hostname}:{port}/{database}?sslmode=disable
```
#### Rockset
The connection string for Rockset is:
@@ -1192,6 +1210,7 @@ rockset://{api key}:@{api server}/{VI ID}
For more complete instructions, we recommend the [Rockset documentation](https://docs.rockset.com/apache-superset/).
#### Snowflake
##### Install Snowflake Driver
@@ -1199,7 +1218,7 @@ For more complete instructions, we recommend the [Rockset documentation](https:/
Follow the steps [here](/docs/configuration/databases#installing-database-drivers) about how to
install new database drivers when setting up Superset locally via docker compose.
```bash
```
echo "snowflake-sqlalchemy" >> ./docker/requirements-local.txt
```
@@ -1232,7 +1251,7 @@ To connect Snowflake with Key Pair Authentication, you need to add the following
***Please note that you need to merge multi-line private key content to one line and insert `\n` between each line***
```json
```
{
"auth_method": "keypair",
"auth_params": {
@@ -1244,7 +1263,7 @@ To connect Snowflake with Key Pair Authentication, you need to add the following
If your private key is stored on server, you can replace "privatekey_body" with “privatekey_path” in parameter.
```json
```
{
"auth_method": "keypair",
"auth_params": {
@@ -1265,6 +1284,7 @@ The connection string for Solr looks like this:
solr://{username}:{password}@{host}:{port}/{server_path}/{collection}[/?use_ssl=true|false]
```
#### Apache Spark SQL
The recommended connector library for Apache Spark SQL [pyhive](https://pypi.org/project/PyHive/).
@@ -1282,13 +1302,12 @@ The recommended connector library for SQL Server is [pymssql](https://github.com
The connection string for SQL Server looks like this:
```
mssql+pymssql://<Username>:<Password>@<Host>:<Port-default:1433>/<Database Name>
mssql+pymssql://<Username>:<Password>@<Host>:<Port-default:1433>/<Database Name>/?Encrypt=yes
```
It is also possible to connect using [pyodbc](https://pypi.org/project/pyodbc) with the parameter [odbc_connect](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#pass-through-exact-pyodbc-string)
The connection string for SQL Server looks like this:
```
mssql+pyodbc:///?odbc_connect=Driver%3D%7BODBC+Driver+17+for+SQL+Server%7D%3BServer%3Dtcp%3A%3Cmy_server%3E%2C1433%3BDatabase%3Dmy_database%3BUid%3Dmy_user_name%3BPwd%3Dmy_password%3BEncrypt%3Dyes%3BConnection+Timeout%3D30
```
@@ -1317,24 +1336,6 @@ starrocks://<User>:<Password>@<Host>:<Port>/<Catalog>.<Database>
StarRocks maintains their Superset docuementation [here](https://docs.starrocks.io/docs/integrations/BI_integrations/Superset/).
:::
#### TDengine
[TDengine](https://www.tdengine.com) is a High-Performance, Scalable Time-Series Database for Industrial IoT and provides SQL-like query interface.
The recommended connector library for TDengine is [taospy](https://pypi.org/project/taospy/) and [taos-ws-py](https://pypi.org/project/taos-ws-py/)
The expected connection string is formatted as follows:
```
taosws://<user>:<password>@<host>:<port>
```
For example:
```
taosws://root:taosdata@127.0.0.1:6041
```
#### Teradata
The recommended connector library is
@@ -1356,7 +1357,7 @@ here: https://downloads.teradata.com/download/connectivity/odbc-driver/linux
Here are the required environment variables:
```bash
```
export ODBCINI=/.../teradata/client/ODBC_64/odbc.ini
export ODBCINST=/.../teradata/client/ODBC_64/odbcinst.ini
```
@@ -1365,8 +1366,8 @@ We recommend using the first library because of the
lack of requirement around ODBC drivers and
because it's more regularly updated.
#### TimescaleDB
#### TimescaleDB
[TimescaleDB](https://www.timescale.com) is the open-source relational database for time-series and analytics to build powerful data-intensive applications.
TimescaleDB is a PostgreSQL extension, and you can use the standard PostgreSQL connector library, [psycopg2](https://www.psycopg.org/docs/), to connect to the database.
@@ -1398,38 +1399,31 @@ postgresql://{username}:{password}@{host}:{port}/{database name}?sslmode=require
[Learn more about TimescaleDB!](https://docs.timescale.com/)
#### Trino
Supported trino version 352 and higher
##### Connection String
The connection string format is as follows:
```
trino://{username}:{password}@{hostname}:{port}/{catalog}
```
If you are running Trino with docker on local machine, please use the following connection URL
```
trino://trino@host.docker.internal:8080
```
##### Authentications
###### 1. Basic Authentication
You can provide `username`/`password` in the connection string or in the `Secure Extra` field at `Advanced / Security`
- In Connection String
* In Connection String
```
trino://{username}:{password}@{hostname}:{port}/{catalog}
```
- In `Secure Extra` field
* In `Secure Extra` field
```json
{
"auth_method": "basic",
@@ -1443,9 +1437,7 @@ You can provide `username`/`password` in the connection string or in the `Secure
NOTE: if both are provided, `Secure Extra` always takes higher priority.
###### 2. Kerberos Authentication
In `Secure Extra` field, config as following example:
```json
{
"auth_method": "kerberos",
@@ -1462,9 +1454,7 @@ All fields in `auth_params` are passed directly to the [`KerberosAuthentication`
NOTE: Kerberos authentication requires installing the [`trino-python-client`](https://github.com/trinodb/trino-python-client) locally with either the `all` or `kerberos` optional features, i.e., installing `trino[all]` or `trino[kerberos]` respectively.
###### 3. Certificate Authentication
In `Secure Extra` field, config as following example:
```json
{
"auth_method": "certificate",
@@ -1478,9 +1468,7 @@ In `Secure Extra` field, config as following example:
All fields in `auth_params` are passed directly to the [`CertificateAuthentication`](https://github.com/trinodb/trino-python-client/blob/0.315.0/trino/auth.py#L416) class.
###### 4. JWT Authentication
Config `auth_method` and provide token in `Secure Extra` field
```json
{
"auth_method": "jwt",
@@ -1491,10 +1479,8 @@ Config `auth_method` and provide token in `Secure Extra` field
```
###### 5. Custom Authentication
To use custom authentication, first you need to add it into
`ALLOWED_EXTRA_AUTHENTICATIONS` allow list in Superset config file:
```python
from your.module import AuthClass
from another.extra import auth_method
@@ -1508,7 +1494,6 @@ ALLOWED_EXTRA_AUTHENTICATIONS: Dict[str, Dict[str, Callable[..., Any]]] = {
```
Then in `Secure Extra` field:
```json
{
"auth_method": "custom_auth",
@@ -1524,8 +1509,8 @@ or factory function (which returns an `Authentication` instance) to `auth_method
All fields in `auth_params` are passed directly to your class/function.
**Reference**:
* [Trino-Superset-Podcast](https://trino.io/episodes/12.html)
- [Trino-Superset-Podcast](https://trino.io/episodes/12.html)
#### Vertica
@@ -1552,6 +1537,8 @@ Other parameters:
- Load Balancer - Backup Host
#### YDB
The recommended connector library for [YDB](https://ydb.tech/) is
@@ -1566,7 +1553,6 @@ ydb://{host}:{port}/{database_name}
```
##### Protocol
You can specify `protocol` in the `Secure Extra` field at `Advanced / Security`:
```
@@ -1577,10 +1563,9 @@ You can specify `protocol` in the `Secure Extra` field at `Advanced / Security`:
Default is `grpc`.
##### Authentication Methods
###### Static Credentials
To use `Static Credentials` you should provide `username`/`password` in the `Secure Extra` field at `Advanced / Security`:
```
@@ -1592,8 +1577,8 @@ To use `Static Credentials` you should provide `username`/`password` in the `Sec
}
```
###### Access Token Credentials
###### Access Token Credentials
To use `Access Token Credentials` you should provide `token` in the `Secure Extra` field at `Advanced / Security`:
```
@@ -1604,8 +1589,8 @@ To use `Access Token Credentials` you should provide `token` in the `Secure Extr
}
```
##### Service Account Credentials
##### Service Account Credentials
To use Service Account Credentials, you should provide `service_account_json` in the `Secure Extra` field at `Advanced / Security`:
```
@@ -1623,6 +1608,8 @@ To use Service Account Credentials, you should provide `service_account_json` in
}
```
#### YugabyteDB
[YugabyteDB](https://www.yugabyte.com/) is a distributed SQL database built on top of PostgreSQL.
@@ -1637,6 +1624,8 @@ The connection string looks like:
postgresql://{username}:{password}@{host}:{port}/{database}
```
## Connecting through the UI
Here is the documentation on how to leverage the new DB Connection UI. This will provide admins the ability to enhance the UX for users who want to connect to new databases.
@@ -1709,6 +1698,9 @@ For databases like MySQL and Postgres that use the standard format of `engine+dr
For other databases you need to implement these methods yourself. The BigQuery DB engine spec is a good example of how to do that.
### Extra Database Settings
##### Deeper SQLAlchemy Integration
@@ -1772,7 +1764,9 @@ You can use the `Extra` field in the **Edit Databases** form to configure SSL:
}
```
## Misc
## Misc.
### Querying across databases

View File

@@ -10,7 +10,7 @@ version: 1
The superset cli allows you to import and export datasources from and to YAML. Datasources include
databases. The data is expected to be organized in the following hierarchy:
```text
```
├──databases
| ├──database_1
| | ├──table_1
@@ -30,13 +30,13 @@ databases. The data is expected to be organized in the following hierarchy:
You can print your current datasources to stdout by running:
```bash
```
superset export_datasources
```
To save your datasources to a ZIP file run:
```bash
```
superset export_datasources -f <filename>
```
@@ -55,7 +55,7 @@ Alternatively, you can export datasources using the UI:
In order to obtain an **exhaustive list of all fields** you can import using the YAML import run:
```bash
```
superset export_datasource_schema
```
@@ -65,13 +65,13 @@ As a reminder, you can use the `-b` flag to include back references.
In order to import datasources from a ZIP file, run:
```bash
```
superset import_datasources -p <path / filename>
```
The optional username flag **-u** sets the user used for the datasource import. The default is 'admin'. Example:
```bash
```
superset import_datasources -p <path / filename> -u 'admin'
```
@@ -81,7 +81,7 @@ superset import_datasources -p <path / filename> -u 'admin'
When using Superset version 4.x.x to import from an older version (2.x.x or 3.x.x) importing is supported as the command `legacy_import_datasources` and expects a JSON or directory of JSONs. The options are `-r` for recursive and `-u` for specifying a user. Example of legacy import without options:
```bash
```
superset legacy_import_datasources -p <path or filename>
```
@@ -89,21 +89,21 @@ superset legacy_import_datasources -p <path or filename>
When using an older Superset version (2.x.x & 3.x.x) of Superset, the command is `import_datasources`. ZIP and YAML files are supported and to switch between them the feature flag `VERSIONED_EXPORT` is used. When `VERSIONED_EXPORT` is `True`, `import_datasources` expects a ZIP file, otherwise YAML. Example:
```bash
```
superset import_datasources -p <path or filename>
```
When `VERSIONED_EXPORT` is `False`, if you supply a path all files ending with **yaml** or **yml** will be parsed. You can apply
additional flags (e.g. to search the supplied path recursively):
```bash
```
superset import_datasources -p <path> -r
```
The sync flag **-s** takes parameters in order to sync the supplied elements with your file. Be
careful this can delete the contents of your meta database. Example:
```bash
```
superset import_datasources -p <path / filename> -s columns,metrics
```
@@ -115,7 +115,7 @@ If you dont supply the sync flag (**-s**) importing will only add and update
E.g. you can add a verbose_name to the column ds in the table random_time_series from the example
datasets by saving the following YAML to file and then running the **import_datasources** command.
```yaml
```
databases:
- database_name: main
tables:

View File

@@ -18,7 +18,8 @@ The following keys in `superset_config.py` can be specified to configure CORS:
- `ENABLE_CORS`: Must be set to `True` in order to enable CORS
- `CORS_OPTIONS`: options passed to Flask-CORS
([documentation](https://flask-cors.readthedocs.io/en/latest/api.html#extension))
([documentation](https://flask-cors.corydolphin.com/en/latest/api.html#extension))
## HTTP headers
@@ -26,6 +27,7 @@ Note that Superset bundles [flask-talisman](https://pypi.org/project/talisman/)
Self-described as a small Flask extension that handles setting HTTP headers that can help
protect against a few common web application security issues.
## HTML Embedding of Dashboards and Charts
There are two ways to embed a dashboard: Using the [SDK](https://www.npmjs.com/package/@superset-ui/embedded-sdk) or embedding a direct link. Note that in the latter case everybody who knows the link is able to access the dashboard.
@@ -37,16 +39,14 @@ This works by first changing the content security policy (CSP) of [flask-talisma
#### Changing flask-talisman CSP
Add to `superset_config.py` the entire `TALISMAN_CONFIG` section from `config.py` and include a `frame-ancestors` section:
```python
TALISMAN_ENABLED = True
TALISMAN_CONFIG = {
"content_security_policy": {
...
"frame-ancestors": ["*.my-domain.com", "*.another-domain.com"],
"frame-ancestors": ["*.my-domain.com", "*.another-domain.com"],
...
```
Restart Superset for this configuration change to take effect.
#### Making a Dashboard Public
@@ -69,7 +69,6 @@ Now anybody can directly access the dashboard's URL. You can embed it in an ifra
>
</iframe>
```
#### Embedding a Chart
A chart's embed code can be generated by going to a chart's edit view and then clicking at the top right on `...` > `Share` > `Embed code`
@@ -86,10 +85,11 @@ SUPERSET_FEATURE_EMBEDDED_SUPERSET=true
## CSRF settings
Similarly, [flask-wtf](https://flask-wtf.readthedocs.io/en/0.15.x/config/) is used to manage
Similarly, [flask-wtf](https://flask-wtf.readthedocs.io/en/0.15.x/config/) is used manage
some CSRF configurations. If you need to exempt endpoints from CSRF (e.g. if you are
running a custom auth postback endpoint), you can add the endpoints to `WTF_CSRF_EXEMPT_LIST`:
## SSH Tunneling
1. Turn on feature flag
@@ -105,11 +105,8 @@ running a custom auth postback endpoint), you can add the endpoints to `WTF_CSRF
3. Verify data is flowing
- Once SSH tunneling has been enabled, go to SQL Lab and write a query to verify data is properly flowing.
## Domain Sharding
:::note
Domain Sharding is deprecated as of Superset 5.0.0, and will be removed in Superset 6.0.0. Please Enable HTTP2 to keep more open connections per domain.
:::
## Domain Sharding
Chrome allows up to 6 open connections per domain at a time. When there are more than 6 slices in
dashboard, a lot of time fetch requests are queued up and wait for next available socket.

View File

@@ -0,0 +1,6 @@
---
title: Setup SSH Tunneling
hide_title: true
sidebar_position: 8
version: 1
---

View File

@@ -77,7 +77,6 @@ In the UI you can assign a set of parameters as JSON
"my_table": "foo"
}
```
The parameters become available in your SQL (example: `SELECT * FROM {{ my_table }}` ) by using Jinja templating syntax.
SQL Lab template parameters are stored with the dataset as `TEMPLATE PARAMETERS`.
@@ -104,6 +103,7 @@ GROUP BY action
Note ``_filters`` is not stored with the dataset. It's only used within the SQL Lab UI.
Besides default Jinja templating, SQL lab also supports self-defined template processor by setting
the `CUSTOM_TEMPLATE_PROCESSORS` in your superset configuration. The values in this dictionary
overwrite the default Jinja template processors of the specified database engine. The example below
@@ -186,7 +186,7 @@ cache hit in the future and Superset can retrieve cached data.
You can disable the inclusion of the `username` value in the calculation of the
cache key by adding the following parameter to your Jinja code:
```python
```
{{ current_username(add_to_cache_keys=False) }}
```
@@ -201,7 +201,7 @@ cache hit in the future and Superset can retrieve cached data.
You can disable the inclusion of the account `id` value in the calculation of the
cache key by adding the following parameter to your Jinja code:
```python
```
{{ current_user_id(add_to_cache_keys=False) }}
```
@@ -216,7 +216,7 @@ cache hit in the future and Superset can retrieve cached data.
You can disable the inclusion of the email value in the calculation of the
cache key by adding the following parameter to your Jinja code:
```python
```
{{ current_user_email(add_to_cache_keys=False) }}
```
@@ -273,7 +273,7 @@ You can retrieve the value for a specific filter as a list using `{{ filter_valu
This is useful if:
- You want to use a filter component to filter a query where the name of filter component column doesn't match the one in the select statement
- You want to have the ability to filter inside the main query for performance purposes
- You want to have the ability for filter inside the main query for performance purposes
Here's a concrete example:
@@ -301,7 +301,7 @@ This is useful if:
Here's a concrete example:
```sql
```
WITH RECURSIVE
superiors(employee_id, manager_id, full_name, level, lineage) AS (
SELECT
@@ -357,7 +357,6 @@ considerably improve performance, as many databases and query engines are able t
if the temporal filter is placed on the inner query, as opposed to the outer query.
The macro takes the following parameters:
- `column`: Name of the temporal column. Leave undefined to reference the time range from a Dashboard Native Time Range
filter (when present).
- `default`: The default value to fall back to if the time filter is not present, or has the value `No filter`
@@ -371,7 +370,6 @@ The macro takes the following parameters:
filter should only apply to the inner query.
The return type has the following properties:
- `from_expr`: the start of the time filter (if any)
- `to_expr`: the end of the time filter (if any)
- `time_range`: The applied time range
@@ -412,7 +410,6 @@ LIMIT 1000;
When using the `default` parameter, the templated query can be simplified, as the endpoints will always be defined
(to use a fixed time range, you can also use something like `default="2024-08-27 : 2024-09-03"`)
```
{% set time_filter = get_time_filter("dttm", default="Last week", remove_filter=True) %}
SELECT
@@ -432,19 +429,19 @@ To use the macro, first you need to find the ID of the dataset. This can be done
Once you have the ID you can query it as if it were a table:
```sql
```
SELECT * FROM {{ dataset(42) }} LIMIT 10
```
If you want to select the metric definitions as well, in addition to the columns, you need to pass an additional keyword argument:
```sql
```
SELECT * FROM {{ dataset(42, include_metrics=True) }} LIMIT 10
```
Since metrics are aggregations, the resulting SQL expression will be grouped by all non-metric columns. You can specify a subset of columns to group by instead:
```sql
```
SELECT * FROM {{ dataset(42, include_metrics=True, columns=["ds", "category"]) }} LIMIT 10
```

View File

@@ -24,7 +24,7 @@ The challenge however lies with the slew of [database engines](/docs/configurati
For example the following is a comparison of MySQL and Presto,
```python
```
import pandas as pd
from sqlalchemy import create_engine
@@ -41,7 +41,7 @@ pd.read_sql_query(
which outputs `{"ts":{"0":1640995200000}}` (which infers the UTC timezone per the Epoch time definition) and `{"ts":{"0":"2022-01-01 00:00:00.000"}}` (without an explicit timezone) respectively and thus are treated differently in JavaScript:
```js
```
new Date(1640995200000)
> Sat Jan 01 2022 13:00:00 GMT+1300 (New Zealand Daylight Time)

View File

@@ -52,7 +52,7 @@ Note that:
[docker-compose.yml](https://github.com/apache/superset/blob/master/docker-compose.yml)
- The local repository is mounted within the services, meaning updating
the code on the host will be reflected in the docker images
- Superset is served at localhost:9000/
- Superset is served at localhost:8088/
- You can login with admin/admin
:::note
@@ -72,36 +72,6 @@ documentation.
configured to be secure.
:::
### Supported environment variables
Affecting the Docker build process:
- **SUPERSET_BUILD_TARGET (default=dev):** which --target to build, either `lean` or `dev` are commonly used
- **INCLUDE_FIREFOX (default=false):** whether to include the Firefox headless browser in the build
- **INCLUDE_CHROMIUM (default=false):** whether to include the Firefox headless browser in the build
- **BUILD_TRANSLATIONS(default=false):** whether to compile the translations from the .po files available
- **SUPERSET_LOAD_EXAMPLES (default=yes):** whether to load the examples into the database upon startup,
save some precious time on startup by `SUPERSET_LOAD_EXAMPLES=no docker compose up`
- **SUPERSET_LOG_LEVEL (default=info)**: Can be set to debug, info, warning, error, critical
for more verbose logging
For more env vars that affect your configuration, see this
[superset_config.py](https://github.com/apache/superset/blob/master/docker/pythonpath_dev/superset_config.py)
used in the `docker compose` context to assign env vars to the superset configuration.
### Accessing the postgres database
Sometimes it's useful to access the database in the docker container directly.
You can enter a `psql` shell (the official Postgres client) by running the following command:
```bash
docker compose exec db psql -U superset
```
Also note that the database is exposed on port 5432, so you can connect to it using your favorite
Postgres client or even SQL Lab itselft directly in Superset by creating a new database connection
to `localhost:5432`.
### Nuking the postgres database
At times, it's possible to end up with your development database in a bad state, it's
@@ -134,8 +104,7 @@ instance, but many people like to run that tooling from their host.
Assuming you already have a way to setup your python environments
like `pyenv`, `virtualenv` or something else, all you should have to
do is to install our dev, pinned python requirements bundle, after installing
the prerequisites mentioned in [OS Dependencies](https://superset.apache.org/docs/installation/pypi/#os-dependencies)
do is to install our dev, pinned python requirements bundle
```bash
pip install -r requirements/development.txt
@@ -270,22 +239,22 @@ If you have made changes to the FAB-managed templates, which are not built the s
If you add a new requirement or update an existing requirement (per the `install_requires` section in `setup.py`) you must recompile (freeze) the Python dependencies to ensure that for CI, testing, etc. the build is deterministic. This can be achieved via,
```bash
python3 -m venv venv
source venv/bin/activate
python3 -m pip install -r requirements/development.txt
./scripts/uv-pip-compile.sh
$ python3 -m venv venv
$ source venv/bin/activate
$ python3 -m pip install -r requirements/development.txt
$ pip-compile-multi --no-upgrade
```
When upgrading the version number of a single package, you should run `./scripts/uv-pip-compile.sh` with the `-P` flag:
When upgrading the version number of a single package, you should run `pip-compile-multi` with the `-P` flag:
```bash
./scripts/uv-pip-compile.sh -P some-package-to-upgrade
$ pip-compile-multi -P my-package
```
To bring all dependencies up to date as per the restrictions defined in `setup.py` and `requirements/*.in`, run `./scripts/uv-pip-compile.sh --upgrade`
To bring all dependencies up to date as per the restrictions defined in `setup.py` and `requirements/*.in`, run pip-compile-multi` without any flags:
```bash
./scripts/uv-pip-compile.sh --upgrade
$ pip-compile-multi
```
This should be done periodically, but it is recommended to do thorough manual testing of the application to ensure no breaking changes have been introduced that aren't caught by the unit and integration tests.
@@ -506,9 +475,41 @@ pre-commit install
A series of checks will now run when you make a git commit.
## Linting
See [how tos](/docs/contributing/howtos#linting)
### Python
We use [Pylint](https://pylint.org/) for linting which can be invoked via:
```bash
pylint
```
In terms of best practices please avoid blanket disabling of Pylint messages globally (via `.pylintrc`) or top-level within the file header, albeit there being a few exceptions. Disabling should occur inline as it prevents masking issues and provides context as to why said message is disabled.
Additionally, the Python code is auto-formatted using [Black](https://github.com/python/black) which
is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/integrations/editors.html)
### TypeScript
```bash
cd superset-frontend
npm ci
# run eslint checks
npm run eslint -- .
# run tsc (typescript) checks
npm run type
```
If using the eslint extension with vscode, put the following in your workspace `settings.json` file:
```json
"eslint.workingDirectories": [
"superset-frontend"
]
```
## GitHub Actions and `act`
@@ -522,7 +523,6 @@ For more targetted iteration, see the `gh workflow run --ref {BRANCH}` subcomman
For automation and CI/CD, Superset makes extensive use of GitHub Actions (GHA). You
can find all of the workflows and other assets under the `.github/` folder. This includes:
- running the backend unit test suites (`tests/`)
- running the frontend test suites (`superset-frontend/src/**.*.test.*`)
- running our Cypress end-to-end tests (`superset-frontend/cypress-base/`)
@@ -564,7 +564,6 @@ act pull_request --job {workflow_name} --secret GITHUB_TOKEN=$GITHUB_TOKEN --con
```
In the example above, notice that:
- we target a specific workflow, using `--job`
- we pass a secret using `--secret`, as many jobs require read access (public) to the repo
- we simulate a `pull_request` event by specifying it as the first arg,
@@ -644,6 +643,71 @@ To run a single test file:
npm run test -- path/to/file.js
```
### Integration Testing
We use [Cypress](https://www.cypress.io/) for integration tests. To open Cypress and explore tests first setup and run test server:
```bash
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export CYPRESS_BASE_URL="http://localhost:8081"
superset db upgrade
superset load_test_users
superset load-examples --load-test-data
superset init
superset run --port 8081
```
Run Cypress tests:
```bash
cd superset-frontend
npm run build-instrumented
cd cypress-base
npm install
# run tests via headless Chrome browser (requires Chrome 64+)
npm run cypress-run-chrome
# run tests from a specific file
npm run cypress-run-chrome -- --spec cypress/e2e/explore/link.test.ts
# run specific file with video capture
npm run cypress-run-chrome -- --spec cypress/e2e/dashboard/index.test.js --config video=true
# to open the cypress ui
npm run cypress-debug
# to point cypress to a url other than the default (http://localhost:8088) set the environment variable before running the script
# e.g., CYPRESS_BASE_URL="http://localhost:9000"
CYPRESS_BASE_URL=<your url> npm run cypress open
```
See [`superset-frontend/cypress_build.sh`](https://github.com/apache/superset/blob/master/superset-frontend/cypress_build.sh).
As an alternative you can use docker compose environment for testing:
Make sure you have added below line to your /etc/hosts file:
`127.0.0.1 db`
If you already have launched Docker environment please use the following command to ensure a fresh database instance:
`docker compose down -v`
Launch environment:
`CYPRESS_CONFIG=true docker compose up --build`
It will serve the backend and frontend on port 8088.
Run Cypress tests:
```bash
cd cypress-base
npm install
npm run cypress open
```
### Debugging Server App
#### Local
@@ -786,7 +850,7 @@ To debug Flask running in POD inside a kubernetes cluster, you'll need to make s
add: ["SYS_PTRACE"]
```
See [set capabilities for a container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container) for more details.
See (set capabilities for a container)[https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container] for more details.
Once the pod is running as root and has the SYS_PTRACE capability it will be able to debug the Flask app.

View File

@@ -10,7 +10,7 @@ version: 1
The latest documentation and tutorial are available at https://superset.apache.org/.
The documentation site is built using [Docusaurus 3](https://docusaurus.io/), a modern
The documentation site is built using [Docusaurus 2](https://docusaurus.io/), a modern
static website generator, the source for which resides in `./docs`.
### Local Development
@@ -223,39 +223,36 @@ To run a single test file:
npm run test -- path/to/file.js
```
### E2E Integration Testing
### e2e Integration Testing
For E2E testing, we recommend that you use a `docker compose` backend
For e2e testing, we recommend that you use a `docker-compose` backed-setup
Alternatively, you can go lower level and set things up in your
development environment by following these steps:
First set up a python/flask backend:
```bash
CYPRESS_CONFIG=true docker compose up --build
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export CYPRESS_BASE_URL="http://localhost:8081"
superset db upgrade
superset load_test_users
superset init
superset load-examples --load-test-data
superset run --port 8081
```
`docker compose` will get to work and expose a Cypress-ready Superset app.
This app uses a different database schema (`superset_cypress`) to keep it isolated from
your other dev environmen(s)t, a specific set of examples, and a set of configurations that
aligns with the expectations within the end-to-end tests. Also note that it's served on a
different port than the default port for the backend (`8088`).
Now in another terminal, let's get ready to execute some Cypress commands. First, tell cypress
to connect to the Cypress-ready Superset backend.
```
CYPRESS_BASE_URL=http://localhost:8081
```
In another terminal, prepare the frontend and run Cypress tests:
```bash
# superset-frontend/cypress-base is the base folder for everything Cypress-related
# It's essentially its own npm app, with its own dependencies, configurations and utilities
cd superset-frontend/cypress-base
cd superset-frontend
npm run build-instrumented
cd cypress-base
npm install
# use interactive mode to run tests, while keeping memory usage contained
# this will fire up an interactive Cypress UI
# as you alter the code, the tests will re-run automatically, and you can visualize each
# and every step for debugging purposes
npx cypress open --config numTestsKeptInMemory=5
# to run the test suite on the command line using chrome (same as CI)
# run tests via headless Chrome browser (requires Chrome 64+)
npm run cypress-run-chrome
# run tests from a specific file
@@ -267,6 +264,9 @@ npm run cypress-run-chrome -- --spec cypress/e2e/dashboard/index.test.js --confi
# to open the cypress ui
npm run cypress-debug
# to point cypress to a url other than the default (http://localhost:8088) set the environment variable before running the script
# e.g., CYPRESS_BASE_URL="http://localhost:9000"
CYPRESS_BASE_URL=<your url> npm run cypress open
```
See [`superset-frontend/cypress_build.sh`](https://github.com/apache/superset/blob/master/superset-frontend/cypress_build.sh).
@@ -411,7 +411,7 @@ See [set capabilities for a container](https://kubernetes.io/docs/tasks/configur
Once the pod is running as root and has the `SYS_PTRACE` capability it will be able to debug the Flask app.
You can follow the same instructions as in `docker compose`. Enter the pod and install the required library and packages: gdb, netstat and debugpy.
You can follow the same instructions as in `docker compose`. Enter the pod and install the required library and packages; gdb, netstat and debugpy.
Often in a Kubernetes environment nodes are not addressable from outside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a Kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
@@ -571,9 +571,13 @@ pybabel compile -d superset/translations
### Python
We use [ruff](https://github.com/astral-sh/ruff) for linting which can be invoked via:
We use [Pylint](https://pylint.org/) and [ruff](https://github.com/astral-sh/ruff)
for linting which can be invoked via:
```
# Run pylint
pylint superset/
# auto-reformat using ruff
ruff format
@@ -584,8 +588,11 @@ ruff check
ruff check --fix
```
Ruff configuration is located in our
(pyproject.toml)[https://github.com/apache/superset/blob/master/pyproject.toml] file
In terms of best practices please avoid blanket disabling of Pylint messages globally
(via `.pylintrc`) or top-level within the file header, albeit there being a few exceptions.
Disabling should occur inline as it prevents masking issues and provides context as to why
said message is disabled.
All this is configured to run in pre-commit hooks, which we encourage you to setup
with `pre-commit install`
@@ -608,27 +615,3 @@ If using the eslint extension with vscode, put the following in your workspace `
"superset-frontend"
]
```
## GitHub Ephemeral Environments
On any given pull request on GitHub, it's possible to create a temporary environment/deployment
by simply adding the label `testenv-up` to the PR. Once you add the `testenv-up` label, a
GitHub Action will be triggered that will:
- build a docker image
- deploy it in EC2 (sponsored by the folks at [Preset](https://preset.io))
- write a comment on the PR with a link to the ephemeral environment
For more advanced use cases, it's possible to set a feature flag on the PR body, which will
take effect on the ephemeral environment. For example, if you want to set the `TAGGING_SYSTEM`
feature flag to `true`, you can add the following line to the PR body/description:
```
FEATURE_TAGGING_SYSTEM=true
```
Simarly, it's possible to disable feature flags with:
```
FEATURE_TAGGING_SYSTEM=false
```

View File

@@ -3,7 +3,7 @@ sidebar_position: 6
version: 1
---
# Miscellaneous
# Misc.
## Reporting a Security Vulnerability
@@ -11,7 +11,7 @@ Please report security vulnerabilities to private@superset.apache.org.
In the event a community member discovers a security flaw in Superset, it is important to follow the [Apache Security Guidelines](https://www.apache.org/security/committers.html) and release a fix as quickly as possible before public disclosure. Reporting security vulnerabilities through the usual GitHub Issues channel is not ideal as it will publicize the flaw before a fix can be applied.
## SQL Lab Async
### SQL Lab Async
It's possible to configure a local database to operate in `async` mode,
to work on `async` related features.
@@ -46,7 +46,7 @@ Note that:
to your production environment, and use the similar broker as well as
results backend configuration
## Async Chart Queries
### Async Chart Queries
It's possible to configure database queries for charts to operate in `async` mode. This is especially useful for dashboards with many charts that may otherwise be affected by browser connection limits. To enable async queries for dashboards and Explore, the following dependencies are required:

View File

@@ -7,7 +7,7 @@ import InteractiveSVG from '../../src/components/InteractiveERDSVG';
# Resources
## Entity-Relationship Diagram
## Entity-Relationship Diagram
Here is our interactive ERD:

View File

@@ -66,7 +66,7 @@ For running long query from Sql Lab, by default Superset allows it run as long a
being killed by celery. If you want to increase the time for running query, you can specify the
timeout in configuration. For example:
```python
```
SQLLAB_ASYNC_TIME_LIMIT_SEC = 60 * 60 * 6
```
@@ -78,7 +78,7 @@ come back within client-side timeout (60 seconds by default), Superset will disp
to avoid gateway timeout message. If you have a longer gateway timeout limit, you can change the
timeout settings in **superset_config.py**:
```python
```
SUPERSET_WEBSERVER_TIMEOUT = 60
```
@@ -87,7 +87,7 @@ SUPERSET_WEBSERVER_TIMEOUT = 60
You need to register a free account at [Mapbox.com](https://www.mapbox.com), obtain an API key, and add it
to **.env** at the key MAPBOX_API_KEY:
```python
```
MAPBOX_API_KEY = "longstringofalphanumer1c"
```
@@ -99,7 +99,7 @@ refreshed - especially if some data is slow moving, or run heavy queries. To exc
from the timed refresh process, add the `timed_refresh_immune_slices` key to the dashboard JSON
Metadata field:
```json
```
{
"filter_immune_slices": [],
"expanded_slices": {},
@@ -115,7 +115,7 @@ Slice refresh will also be staggered over the specified period. You can turn off
setting the `stagger_refresh` to false and modify the stagger period by setting `stagger_time` to a
value in milliseconds in the JSON Metadata field:
```json
```
{
"stagger_refresh": false,
"stagger_time": 2500
@@ -125,7 +125,7 @@ value in milliseconds in the JSON Metadata field:
Here, the entire dashboard will refresh at once if periodic refresh is on. The stagger time of 2.5
seconds is ignored.
**Why does flask fab or superset freeze/hang/not responding when started (my home directory is
**Why does flask fab or superset freezed/hung/not responding when started (my home directory is
NFS mounted)?**
By default, Superset creates and uses an SQLite database at `~/.superset/superset.db`. SQLite is
@@ -137,7 +137,7 @@ You can override this path using the **SUPERSET_HOME** environment variable.
Another workaround is to change where superset stores the sqlite database by adding the following in
`superset_config.py`:
```python
```
SQLALCHEMY_DATABASE_URI = 'sqlite:////new/location/superset.db?check_same_thread=false'
```
@@ -157,12 +157,12 @@ table afterwards to configure the Columns tab, check the appropriate boxes and s
To clarify, the database backend is an OLTP database used by Superset to store its internal
information like your list of users and dashboard definitions. While Superset supports a
[variety of databases as data _sources_](/docs/configuration/databases#installing-database-drivers),
[variety of databases as data *sources*](/docs/configuration/databases#installing-database-drivers),
only a few database engines are supported for use as the OLTP backend / metadata store.
Superset is tested using MySQL, PostgreSQL, and SQLite backends. Its recommended you install
Superset on one of these database servers for production. Installation on other OLTP databases
may work but isnt tested. It has been reported that [Microsoft SQL Server does _not_
may work but isnt tested. It has been reported that [Microsoft SQL Server does *not*
work as a Superset backend](https://github.com/apache/superset/issues/18961). Column-store,
non-OLTP databases are not designed for this type of workload.
@@ -213,7 +213,7 @@ SQLAlchemy and DBAPI scope. This includes features like:
Beyond the SQLAlchemy connector, its also possible, though much more involved, to extend Superset
and write your own connector. The only example of this at the moment is the Druid connector, which
is getting superseded by Druids growing SQL support and the recent availability of a DBAPI and
SQLAlchemy driver. If the database you are considering integrating has any kind of SQL support,
SQLAlchemy driver. If the database you are considering integrating has any kind of of SQL support,
its probably preferable to go the SQLAlchemy route. Note that for a native connector to be possible
the database needs to have support for running OLAP-type queries and should be able to do things that
are typical in basic SQL:
@@ -236,7 +236,7 @@ made to cover more and more use cases.
The API available is documented using [Swagger](https://swagger.io/) and the documentation can be
made available under **/swagger/v1** by enabling the following flag in `superset_config.py`:
```python
```
FAB_API_SWAGGER_UI = True
```

View File

@@ -14,7 +14,6 @@ This page is meant to give new administrators an understanding of Superset's com
## Components
A Superset installation is made up of these components:
1. The Superset application itself
2. A metadata database
3. A caching layer (optional, but necessary for some features)
@@ -23,7 +22,6 @@ A Superset installation is made up of these components:
### Optional components and associated features
The optional components above are necessary to enable these features:
- [Alerts and Reports](/docs/configuration/alerts-reports)
- [Caching](/docs/configuration/cache)
- [Async Queries](/docs/configuration/async-queries-celery/)
@@ -38,7 +36,6 @@ Here are further details on each component.
### The Superset Application
This is the core application. Superset operates like this:
- A user visits a chart or dashboard
- That triggers a SQL query to the data warehouse holding the underlying dataset
- The resulting data is served up in a data visualization
@@ -48,14 +45,13 @@ This is the core application. Superset operates like this:
This is where chart and dashboard definitions, user information, logs, etc. are stored. Superset is tested to work with PostgreSQL and MySQL databases as the metadata database (not be confused with a data source like your data warehouse, which could be a much greater variety of options like Snowflake, Redshift, etc.).
Some installation methods like our Quickstart and PyPI come configured by default to use a SQLite on-disk database. And in a Docker Compose installation, the data would be stored in a PostgreSQL container volume. Neither of these cases are recommended for production instances of Superset.
Some installation methods like our Quickstart and PyPI come configured by default to use a SQLite on-disk database. And in a Docker Compose installation, the data would be stored in a PostgresQL container volume. Neither of these cases are recommended for production instances of Superset.
For production, a properly-configured, managed, standalone database is recommended. No matter what database you use, you should plan to back it up regularly.
### Caching Layer
The caching layer serves two main functions:
- Store the results of queries to your data warehouse so that when a chart is loaded twice, it pulls from the cache the second time, speeding up the application and reducing load on your data warehouse.
- Act as a message broker for the worker, enabling the Alerts & Reports, async queries, and thumbnail caching features.

View File

@@ -59,13 +59,14 @@ Here are the build presets that are exposed through the `supersetbot docker` uti
this specific SHA, which could be from a `master` merge, or release.
- `websocket-latest`: The WebSocket image for use in a Superset cluster.
For insights or modifications to the build matrix and tagging conventions,
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
GitHub action.
## Key ARGs in Dockerfile
- `BUILD_TRANSLATIONS`: whether to build the translations into the image. For the
frontend build this tells webpack to strip out all locales other than `en` from
the `moment-timezone` library. For the backendthis skips compiling the

Some files were not shown because too many files have changed in this diff Show More