diff --git a/.asf.yaml b/.asf.yaml index e717a33715e..3941dae4ff9 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -77,23 +77,17 @@ github: # combination here. contexts: - lint-check - - cypress-matrix (0, chrome) - - cypress-matrix (1, chrome) - - cypress-matrix (2, chrome) - - cypress-matrix (3, chrome) - - cypress-matrix (4, chrome) - - cypress-matrix (5, chrome) + - cypress-matrix-required - dependency-review - frontend-build - - playwright-tests (chromium) + - playwright-tests-required - pre-commit (current) - - pre-commit (previous) - test-mysql - - test-postgres (current) + - test-postgres-required - test-postgres-hive - test-postgres-presto - test-sqlite - - unit-tests (current) + - unit-tests-required required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 15e7a6b14cd..6c8165357f9 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -41,8 +41,8 @@ body: label: Superset version options: - master / latest-dev + - "6.1.0" - "6.0.0" - - "5.0.0" validations: required: true - type: dropdown diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fc0ffbd8561..458cb5bfd9f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,12 +14,6 @@ updates: - package-ecosystem: "npm" ignore: - # TODO: remove below entries until React >= 18.0.0 - - dependency-name: "storybook" - update-types: ["version-update:semver-major", "version-update:semver-minor"] - - dependency-name: "@storybook*" - update-types: ["version-update:semver-major", "version-update:semver-minor"] - - dependency-name: "eslint-plugin-storybook" - dependency-name: "react-error-boundary" - dependency-name: "@rjsf/*" # remark-gfm v4+ requires react-markdown v9+, which needs React 18 @@ -42,14 +36,6 @@ updates: # and confirm the issue https://github.com/apache/superset/issues/39600 is fixed - dependency-name: "react-checkbox-tree" update-types: ["version-update:semver-major"] - groups: - storybook: - applies-to: version-updates - patterns: - - "@storybook*" - - "storybook" - update-types: - - "patch" directory: "/superset-frontend/" schedule: interval: "daily" @@ -90,21 +76,7 @@ updates: - package-ecosystem: "npm" directory: "/docs/" ignore: - # TODO: remove below entries until React >= 18.0.0 in superset-frontend - - dependency-name: "storybook" - update-types: ["version-update:semver-major", "version-update:semver-minor"] - - dependency-name: "@storybook*" - update-types: ["version-update:semver-major", "version-update:semver-minor"] - - dependency-name: "eslint-plugin-storybook" - dependency-name: "react-error-boundary" - groups: - storybook: - applies-to: version-updates - patterns: - - "@storybook*" - - "storybook" - update-types: - - "patch" schedule: interval: "daily" open-pull-requests-limit: 10 diff --git a/.github/workflows/bump-python-package.yml b/.github/workflows/bump-python-package.yml index d69b17d09e5..2ad6ab73476 100644 --- a/.github/workflows/bump-python-package.yml +++ b/.github/workflows/bump-python-package.yml @@ -30,9 +30,8 @@ jobs: pull-requests: write checks: write steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: true ref: master diff --git a/.github/workflows/check-python-deps.yml b/.github/workflows/check-python-deps.yml index ca1d3eb20b6..dc0f3738402 100644 --- a/.github/workflows/check-python-deps.yml +++ b/.github/workflows/check-python-deps.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/check_db_migration_confict.yml b/.github/workflows/check_db_migration_confict.yml index 88953d505d3..a2e6812a8f4 100644 --- a/.github/workflows/check_db_migration_confict.yml +++ b/.github/workflows/check_db_migration_confict.yml @@ -25,7 +25,7 @@ jobs: pull-requests: write steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Check and notify diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index b6355915362..84f90b72a8b 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -75,14 +75,14 @@ jobs: issues: write id-token: write steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - fetch-depth: 1 + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + fetch-depth: 1 - - name: Run Claude PR Action - uses: anthropics/claude-code-action@5fb899572b81d2bb648d4d187173a2f423a9677c # beta - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - timeout_minutes: "60" + - name: Run Claude PR Action + uses: anthropics/claude-code-action@5fb899572b81d2bb648d4d187173a2f423a9677c # beta + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + timeout_minutes: "60" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 58121e3a581..4c104344e9f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -15,9 +15,35 @@ concurrency: cancel-in-progress: true jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + frontend: ${{ steps.check.outputs.frontend }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + analyze: name: Analyze + needs: changes + # Skip on PRs that touch neither code group (e.g. docs-only) so the + # analysis runners don't spin up. push/schedule runs always proceed: + # the change-detector returns "all changed" for non-PR events. + if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 30 permissions: actions: read contents: read @@ -31,19 +57,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4 + uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +74,6 @@ jobs: # queries: security-extended,security-and-quality - name: Perform CodeQL Analysis - if: steps.check.outputs.python || steps.check.outputs.frontend - uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4 + uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ab8777b3762..b595604cc58 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout Repository" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: "Dependency Review" @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: "Checkout Repository" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 558a285f770..c305fab4c13 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,9 +18,30 @@ concurrency: cancel-in-progress: true jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + frontend: ${{ steps.check.outputs.frontend }} + docker: ${{ steps.check.outputs.docker }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} setup_matrix: runs-on: ubuntu-24.04 + timeout-minutes: 5 outputs: matrix_config: ${{ steps.set_matrix.outputs.matrix_config }} steps: @@ -32,8 +53,13 @@ jobs: docker-build: name: docker-build - needs: setup_matrix + needs: [setup_matrix, changes] + if: >- + needs.changes.outputs.python == 'true' || + needs.changes.outputs.frontend == 'true' || + needs.changes.outputs.docker == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 60 strategy: matrix: build_preset: ${{fromJson(needs.setup_matrix.outputs.matrix_config)}} @@ -44,20 +70,12 @@ jobs: IMAGE_TAG: apache/superset:GHA-${{ matrix.build_preset }}-${{ github.run_id }} steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Docker Environment - if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker uses: ./.github/actions/setup-docker with: dockerhub-user: ${{ secrets.DOCKERHUB_USER }} @@ -65,11 +83,9 @@ jobs: build: "true" - name: Setup supersetbot - if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker uses: ./.github/actions/setup-supersetbot/ - name: Build Docker Image - if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -95,7 +111,7 @@ jobs: # in the context of push (using multi-platform build), we need to pull the image locally - name: Docker pull - if: github.event_name == 'push' && (steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker) + if: github.event_name == 'push' run: | for i in 1 2 3; do docker pull $IMAGE_TAG && break @@ -103,7 +119,6 @@ jobs: done - name: Print docker stats - if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker run: | echo "SHA: ${{ github.sha }}" echo "IMAGE: $IMAGE_TAG" @@ -111,7 +126,7 @@ jobs: docker history $IMAGE_TAG - name: docker-compose sanity check - if: (steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker) && matrix.build_preset == 'dev' + if: matrix.build_preset == 'dev' shell: bash env: BUILD_PRESET: ${{ matrix.build_preset }} @@ -124,20 +139,16 @@ jobs: docker-compose-image-tag: # Run this job only on pushes to master (not for PRs) # goal is to check that building the latest image works, not required for all PR pushes - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: changes + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.changes.outputs.docker == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 30 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Docker Environment - if: steps.check.outputs.docker uses: ./.github/actions/setup-docker with: dockerhub-user: ${{ secrets.DOCKERHUB_USER }} @@ -145,7 +156,6 @@ jobs: build: "false" install-docker-compose: "true" - name: docker-compose sanity check - if: steps.check.outputs.docker shell: bash run: | docker compose -f docker-compose-image-tag.yml up superset-init --exit-code-from superset-init diff --git a/.github/workflows/embedded-sdk-release.yml b/.github/workflows/embedded-sdk-release.yml index 1373a72d650..6404284335a 100644 --- a/.github/workflows/embedded-sdk-release.yml +++ b/.github/workflows/embedded-sdk-release.yml @@ -33,13 +33,13 @@ jobs: run: working-directory: superset-embedded-sdk steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-embedded-sdk/.nvmrc' - registry-url: 'https://registry.npmjs.org' + node-version-file: "./superset-embedded-sdk/.nvmrc" + registry-url: "https://registry.npmjs.org" - run: npm ci - run: npm run ci:release env: diff --git a/.github/workflows/embedded-sdk-test.yml b/.github/workflows/embedded-sdk-test.yml index d59254423b5..d7116e8bda3 100644 --- a/.github/workflows/embedded-sdk-test.yml +++ b/.github/workflows/embedded-sdk-test.yml @@ -21,13 +21,13 @@ jobs: run: working-directory: superset-embedded-sdk steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-embedded-sdk/.nvmrc' - registry-url: 'https://registry.npmjs.org' + node-version-file: "./superset-embedded-sdk/.nvmrc" + registry-url: "https://registry.npmjs.org" - run: npm ci - run: npm test - run: npm run build diff --git a/.github/workflows/generate-FOSSA-report.yml b/.github/workflows/generate-FOSSA-report.yml index fe1000a4844..07af069ea30 100644 --- a/.github/workflows/generate-FOSSA-report.yml +++ b/.github/workflows/generate-FOSSA-report.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/github-action-validator.yml b/.github/workflows/github-action-validator.yml index 32718ba6936..912efdcacb4 100644 --- a/.github/workflows/github-action-validator.yml +++ b/.github/workflows/github-action-validator.yml @@ -12,8 +12,12 @@ on: permissions: contents: read -jobs: +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true +jobs: validate-all-ghas: runs-on: ubuntu-24.04 permissions: @@ -23,14 +27,14 @@ jobs: security-events: write steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: "20" - name: Install Dependencies run: npm install -g @action-validator/core @action-validator/cli --save-dev diff --git a/.github/workflows/issue_creation.yml b/.github/workflows/issue_creation.yml index a0d77f0fc0a..f0be6ea3e3b 100644 --- a/.github/workflows/issue_creation.yml +++ b/.github/workflows/issue_creation.yml @@ -15,9 +15,8 @@ jobs: pull-requests: write issues: write steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 2d83a891f15..e5ac346bcbc 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -2,6 +2,11 @@ name: "Pull Request Labeler" on: - pull_request_target +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: labeler: permissions: diff --git a/.github/workflows/latest-release-tag.yml b/.github/workflows/latest-release-tag.yml index 0d1adc42da7..fd85622f1ba 100644 --- a/.github/workflows/latest-release-tag.yml +++ b/.github/workflows/latest-release-tag.yml @@ -11,29 +11,29 @@ jobs: contents: write steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - submodules: recursive + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + submodules: recursive - - name: Check for latest tag - id: latest-tag - env: - RELEASE_TAG_NAME: ${{ github.event.release.tag_name }} - run: | - source ./scripts/tag_latest_release.sh "$RELEASE_TAG_NAME" --dry-run + - name: Check for latest tag + id: latest-tag + env: + RELEASE_TAG_NAME: ${{ github.event.release.tag_name }} + run: | + source ./scripts/tag_latest_release.sh "$RELEASE_TAG_NAME" --dry-run - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - name: Run latest-tag - uses: ./.github/actions/latest-tag - if: steps.latest-tag.outputs.SKIP_TAG != 'true' - with: - description: Superset latest release - tag-name: latest - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Run latest-tag + uses: ./.github/actions/latest-tag + if: steps.latest-tag.outputs.SKIP_TAG != 'true' + with: + description: Superset latest release + tag-name: latest + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 775e08d0ca8..fba2174f21b 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -18,14 +18,14 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - name: Setup Java uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: - distribution: 'temurin' - java-version: '11' + distribution: "temurin" + java-version: "11" - name: Run license check run: ./scripts/check_license.sh diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml index 8a02c6afc36..f6b1b28c069 100644 --- a/.github/workflows/pr-lint.yml +++ b/.github/workflows/pr-lint.yml @@ -8,6 +8,11 @@ on: # Possible values: https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request types: [opened, edited, reopened, synchronize] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: lint-check: runs-on: ubuntu-24.04 @@ -16,7 +21,7 @@ jobs: pull-requests: write steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive @@ -26,6 +31,5 @@ jobs: on-failed-regex-fail-action: true on-failed-regex-request-changes: false on-failed-regex-create-review: false - on-failed-regex-comment: - "Please format your PR title to match: `%regex%`!" + on-failed-regex-comment: "Please format your PR title to match: `%regex%`!" repo-token: "${{ github.token }}" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index de70c687eb7..39a16c3fb78 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -19,12 +19,16 @@ concurrency: jobs: pre-commit: runs-on: ubuntu-24.04 + timeout-minutes: 20 strategy: matrix: - python-version: ["current", "previous", "next"] + # Run the full version spread on push (master/release) and nightly, + # but only the current version on PRs — lint/format/type results + # rarely differ across patch versions, so 3x per PR is wasteful. + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "previous", "next"]') }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive @@ -44,7 +48,9 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version: '20' + node-version-file: "superset-frontend/.nvmrc" + cache: "npm" + cache-dependency-path: "superset-frontend/package-lock.json" - name: Install Frontend Dependencies run: | @@ -68,7 +74,7 @@ jobs: id: changed_files uses: ./.github/actions/file-changes-action with: - output: ' ' + output: " " - name: pre-commit env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28f81cd2c08..8cb5738e1e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false # pulls all commits (needed for lerna / semantic release to correctly version) @@ -52,7 +52,7 @@ jobs: if: env.HAS_TAGS uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" - name: Cache npm if: env.HAS_TAGS diff --git a/.github/workflows/showtime-trigger.yml b/.github/workflows/showtime-trigger.yml index 4bff0497205..12ccb698cd1 100644 --- a/.github/workflows/showtime-trigger.yml +++ b/.github/workflows/showtime-trigger.yml @@ -10,11 +10,11 @@ on: workflow_dispatch: inputs: pr_number: - description: 'PR number to sync' + description: "PR number to sync" required: true type: number sha: - description: 'Specific SHA to deploy (optional, defaults to latest)' + description: "Specific SHA to deploy (optional, defaults to latest)" required: false type: string @@ -152,7 +152,7 @@ jobs: - name: Checkout PR code (only if build needed) if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.build_needed == 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ steps.check.outputs.target_sha }} persist-credentials: false diff --git a/.github/workflows/superset-app-cli.yml b/.github/workflows/superset-app-cli.yml index 0340a0bf045..8ebb2e6539c 100644 --- a/.github/workflows/superset-app-cli.yml +++ b/.github/workflows/superset-app-cli.yml @@ -41,7 +41,7 @@ jobs: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/superset-docs-deploy.yml b/.github/workflows/superset-docs-deploy.yml index 8932e51f4fb..20c116fc28d 100644 --- a/.github/workflows/superset-docs-deploy.yml +++ b/.github/workflows/superset-docs-deploy.yml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout ${{ github.event.workflow_run.head_sha || github.sha }}" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.event.workflow_run.head_sha || github.sha }} persist-credentials: false @@ -68,13 +68,13 @@ jobs: - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './docs/.nvmrc' + node-version-file: "./docs/.nvmrc" - name: Setup Python uses: ./.github/actions/setup-backend/ - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: - distribution: 'zulu' - java-version: '21' + distribution: "zulu" + java-version: "21" - name: Install Graphviz run: sudo apt-get install -y graphviz - name: Compute Entity Relationship diagram (ERD) diff --git a/.github/workflows/superset-docs-verify.yml b/.github/workflows/superset-docs-verify.yml index 72f87014b07..50cac53e7e0 100644 --- a/.github/workflows/superset-docs-verify.yml +++ b/.github/workflows/superset-docs-verify.yml @@ -28,12 +28,12 @@ jobs: name: Link Checking runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false # Do not bump this linkinator-action version without opening # an ASF Infra ticket to allow the new version first! - - uses: JustinBeckwith/linkinator-action@af984b9f30f63e796ae2ea5be5e07cb587f1bbd9 # v2.3 + - uses: JustinBeckwith/linkinator-action@af984b9f30f63e796ae2ea5be5e07cb587f1bbd9 # v2.3 continue-on-error: true # This will make the job advisory (non-blocking, no red X) with: paths: "**/*.md, **/*.mdx" @@ -73,14 +73,14 @@ jobs: working-directory: docs steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './docs/.nvmrc' + node-version-file: "./docs/.nvmrc" - name: yarn install run: | yarn install --check-cache @@ -112,7 +112,7 @@ jobs: working-directory: docs steps: - name: "Checkout PR head: ${{ github.event.workflow_run.head_sha }}" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.event.workflow_run.head_sha }} persist-credentials: false @@ -120,7 +120,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './docs/.nvmrc' + node-version-file: "./docs/.nvmrc" - name: yarn install run: | yarn install --check-cache @@ -131,7 +131,7 @@ jobs: run_id: ${{ github.event.workflow_run.id }} name: database-diagnostics path: docs/src/data/ - if_no_artifact_found: 'warning' + if_no_artifact_found: "warning" - name: Use fresh diagnostics run: | if [ -f "src/data/databases-diagnostics.json" ]; then diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index e686b371257..32761d8e03d 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -10,27 +10,50 @@ on: workflow_dispatch: inputs: use_dashboard: - description: 'Use Cypress Dashboard (true/false) [paid service - trigger manually when needed]. You MUST provide a branch and/or PR number below for this to work.' + description: "Use Cypress Dashboard (true/false) [paid service - trigger manually when needed]. You MUST provide a branch and/or PR number below for this to work." required: false - default: 'false' + default: "false" ref: - description: 'The branch or tag to checkout' + description: "The branch or tag to checkout" required: false - default: '' + default: "" pr_id: - description: 'The pull request ID to checkout' + description: "The pull request ID to checkout" required: false - default: '' + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + frontend: ${{ steps.check.outputs.frontend }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + cypress-matrix: + needs: changes + if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true' # Somehow one test flakes on 24.04 for unknown reasons, this is the only GHA left on 22.04 runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false + timeout-minutes: 30 permissions: contents: read pull-requests: read @@ -41,9 +64,14 @@ jobs: # https://github.com/cypress-io/github-action/issues/48 fail-fast: false matrix: - parallel_id: [0, 1, 2, 3, 4, 5] + parallel_id: [0, 1] browser: ["chrome"] app_root: ${{ github.event_name == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }} + # The /app/prefix variant (push events only) is smoke-tested on a single + # shard rather than the full matrix, so exclude it from the other shards. + exclude: + - parallel_id: 1 + app_root: "/app/prefix" env: SUPERSET_ENV: development SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -70,71 +98,60 @@ jobs: # Conditional checkout based on context - name: Checkout for push or pull_request event if: github.event_name == 'push' || github.event_name == 'pull_request' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Checkout using ref (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} submodules: recursive - name: Checkout using PR ID (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: refs/pull/${{ github.event.inputs.pr_id }}/merge submodules: recursive # ------------------------------------------------------- - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python || steps.check.outputs.frontend - name: Setup postgres - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Import test data - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: testdata - name: Setup Node.js - if: steps.check.outputs.python || steps.check.outputs.frontend uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" + cache: "npm" + cache-dependency-path: "superset-frontend/package-lock.json" - name: Install npm dependencies - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: Build javascript packages - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-instrumented-assets - name: Install cypress - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: cypress-install - name: Run Cypress - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: CYPRESS_BROWSER: ${{ matrix.browser }} PARALLEL_ID: ${{ matrix.parallel_id }} - PARALLELISM: 6 + PARALLELISM: 2 CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} NODE_OPTIONS: "--max-old-space-size=4096" with: @@ -155,7 +172,10 @@ jobs: name: cypress-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}-${{ matrix.parallel_id }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} playwright-tests: + needs: changes + if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true' runs-on: ubuntu-22.04 + timeout-minutes: 30 permissions: contents: read pull-requests: read @@ -188,71 +208,59 @@ jobs: # Conditional checkout based on context (same as Cypress workflow) - name: Checkout for push or pull_request event if: github.event_name == 'push' || github.event_name == 'pull_request' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Checkout using ref (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} submodules: recursive - name: Checkout using PR ID (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: refs/pull/${{ github.event.inputs.pr_id }}/merge submodules: recursive # ------------------------------------------------------- - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python || steps.check.outputs.frontend - name: Setup postgres - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Import test data - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: playwright_testdata - name: Setup Node.js - if: steps.check.outputs.python || steps.check.outputs.frontend uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" + cache: "npm" + cache-dependency-path: "superset-frontend/package-lock.json" - name: Install npm dependencies - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: Build javascript packages - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-instrumented-assets - name: Build embedded SDK - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-embedded-sdk - name: Install Playwright - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: playwright-install - name: Run Playwright (Required Tests) - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: NODE_OPTIONS: "--max-old-space-size=4096" @@ -274,3 +282,63 @@ jobs: ${{ github.workspace }}/superset-frontend/playwright-results/ ${{ github.workspace }}/superset-frontend/test-results/ name: playwright-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} + + # Stable required-status-check anchors. cypress-matrix and playwright-tests + # are matrix jobs gated on change detection (python || frontend). On a PR + # that touches neither — e.g. a docs-only PR — they are skipped at the job + # level, which happens before matrix expansion, so the per-combination + # contexts (`cypress-matrix (0, chrome)`, `playwright-tests (chromium)`) are + # never produced and branch protection waits on them forever. These + # always-running jobs report a single stable context that passes when the + # underlying matrix job succeeded or was skipped, and fails only on a real + # failure. Require these in .asf.yaml instead of the matrix-expanded names. + # + # A matrix job reads as "skipped" in two distinct cases, and only the first + # is a legitimate pass: (a) change detection succeeded and gated the job off + # (docs-only PR); (b) the `changes` job itself failed or was cancelled, in + # which case GHA skips its dependents too. Accepting (b) would let a broken + # change-detector report a false green, so each anchor first requires + # `changes` to have succeeded before honouring a skip. + cypress-matrix-required: + needs: [changes, cypress-matrix] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Check cypress-matrix result + env: + CHANGES: ${{ needs.changes.result }} + RESULT: ${{ needs.cypress-matrix.result }} + run: | + if [ "$CHANGES" != "success" ]; then + echo "change detection did not succeed (result: $CHANGES); refusing to pass on a skipped matrix" + exit 1 + fi + if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then + echo "cypress-matrix did not pass (result: $RESULT)" + exit 1 + fi + echo "cypress-matrix result: $RESULT (changes: $CHANGES)" + + playwright-tests-required: + needs: [changes, playwright-tests] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Check playwright-tests result + env: + CHANGES: ${{ needs.changes.result }} + RESULT: ${{ needs.playwright-tests.result }} + run: | + if [ "$CHANGES" != "success" ]; then + echo "change detection did not succeed (result: $CHANGES); refusing to pass on a skipped matrix" + exit 1 + fi + if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then + echo "playwright-tests did not pass (result: $RESULT)" + exit 1 + fi + echo "playwright-tests result: $RESULT (changes: $CHANGES)" diff --git a/.github/workflows/superset-extensions-cli.yml b/.github/workflows/superset-extensions-cli.yml index a6b361c99f0..8072b7c4c5a 100644 --- a/.github/workflows/superset-extensions-cli.yml +++ b/.github/workflows/superset-extensions-cli.yml @@ -20,15 +20,18 @@ concurrency: jobs: test-superset-extensions-cli-package: runs-on: ubuntu-24.04 + timeout-minutes: 30 strategy: matrix: - python-version: ["previous", "current", "next"] + # Full version spread on push (master/release) + nightly; current only + # on PRs to cut runner cost (cross-version breaks are caught at merge). + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["previous", "current", "next"]') }} defaults: run: working-directory: superset-extensions-cli steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 7ccb4a8709d..87c183aec9d 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -22,11 +22,12 @@ permissions: jobs: frontend-build: runs-on: ubuntu-24.04 + timeout-minutes: 30 outputs: should-run: ${{ steps.check.outputs.frontend }} steps: - name: Checkout Code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false fetch-depth: 0 @@ -74,6 +75,7 @@ jobs: shard: [1, 2, 3, 4, 5, 6, 7, 8] fail-fast: false runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: Download Docker Image Artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -103,11 +105,12 @@ jobs: needs: [sharded-jest-tests] if: needs.frontend-build.outputs.should-run == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 15 permissions: id-token: write steps: - name: Checkout Code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false fetch-depth: 0 @@ -144,6 +147,7 @@ jobs: needs: frontend-build if: needs.frontend-build.outputs.should-run == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: Download Docker Image Artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -168,6 +172,7 @@ jobs: needs: frontend-build if: needs.frontend-build.outputs.should-run == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: Download Docker Image Artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -187,6 +192,7 @@ jobs: needs: frontend-build if: needs.frontend-build.outputs.should-run == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 25 steps: - name: Download Docker Image Artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 diff --git a/.github/workflows/superset-helm-lint.yml b/.github/workflows/superset-helm-lint.yml index f76a39b2cde..9e92b6c9ed7 100644 --- a/.github/workflows/superset-helm-lint.yml +++ b/.github/workflows/superset-helm-lint.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive @@ -33,7 +33,7 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ with: - install-superset: 'false' + install-superset: "false" - name: Set up chart-testing uses: ./.github/actions/chart-testing-action diff --git a/.github/workflows/superset-helm-release.yml b/.github/workflows/superset-helm-release.yml index 29364b46926..fb21407d529 100644 --- a/.github/workflows/superset-helm-release.yml +++ b/.github/workflows/superset-helm-release.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.ref || github.ref_name }} persist-credentials: true diff --git a/.github/workflows/superset-playwright.yml b/.github/workflows/superset-playwright.yml index c676078eb69..5bac1c12b1a 100644 --- a/.github/workflows/superset-playwright.yml +++ b/.github/workflows/superset-playwright.yml @@ -10,23 +10,46 @@ on: workflow_dispatch: inputs: ref: - description: 'The branch or tag to checkout' + description: "The branch or tag to checkout" required: false - default: '' + default: "" pr_id: - description: 'The pull request ID to checkout' + description: "The pull request ID to checkout" required: false - default: '' + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + frontend: ${{ steps.check.outputs.frontend }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + # NOTE: Required Playwright tests are in superset-e2e.yml (E2E / playwright-tests) # This workflow contains only experimental tests that run in shadow mode playwright-tests-experimental: + needs: changes + if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true' runs-on: ubuntu-22.04 + timeout-minutes: 30 continue-on-error: true permissions: contents: read @@ -60,78 +83,65 @@ jobs: # Conditional checkout based on context (same as Cypress workflow) - name: Checkout for push or pull_request event if: github.event_name == 'push' || github.event_name == 'pull_request' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Checkout using ref (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} submodules: recursive - name: Checkout using PR ID (workflow_dispatch) if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != '' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false ref: refs/pull/${{ github.event.inputs.pr_id }}/merge submodules: recursive # ------------------------------------------------------- - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python || steps.check.outputs.frontend - name: Setup postgres - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Import test data - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: playwright_testdata - name: Setup Node.js - if: steps.check.outputs.python || steps.check.outputs.frontend uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" + cache: "npm" + cache-dependency-path: "superset-frontend/package-lock.json" - name: Install npm dependencies - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: Build javascript packages - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-instrumented-assets - name: Build embedded SDK - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-embedded-sdk - name: Install Playwright - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: playwright-install - name: Run Playwright (Experimental Tests) - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: NODE_OPTIONS: "--max-old-space-size=4096" with: run: playwright-run "${{ matrix.app_root }}" experimental/ - name: Run Playwright (Embedded Tests) - if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: NODE_OPTIONS: "--max-old-space-size=4096" diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 5cfcc056fb6..bfa9ef2960d 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -14,8 +14,30 @@ concurrency: cancel-in-progress: true jobs: - test-mysql: + changes: runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + + test-mysql: + needs: changes + if: needs.changes.outputs.python == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 permissions: id-token: write env: @@ -29,9 +51,6 @@ jobs: image: mysql:8.0 # Authenticated pulls use our higher Docker Hub rate limit. Empty on # fork PRs (secrets unavailable) -> runner falls back to anonymous. - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} env: MYSQL_ROOT_PASSWORD: root ports: @@ -43,38 +62,26 @@ jobs: --health-retries=5 redis: image: redis:7-alpine - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} options: --entrypoint redis-server ports: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python - name: Setup MySQL - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: setup-mysql - name: Start Celery worker - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: celery-worker - name: Python integration tests (MySQL) - if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage @@ -85,7 +92,6 @@ jobs: use_oidc: true slug: apache/superset - name: Generate database diagnostics for docs - if: steps.check.outputs.python env: SUPERSET_CONFIG: tests.integration_tests.superset_test_config SUPERSET__SQLALCHEMY_DATABASE_URI: | @@ -108,19 +114,23 @@ jobs: print(f'Generated diagnostics for {len(docs)} databases') " - name: Upload database diagnostics artifact - if: steps.check.outputs.python uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: database-diagnostics path: databases-diagnostics.json retention-days: 7 test-postgres: + needs: changes + if: needs.changes.outputs.python == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 45 permissions: id-token: write strategy: matrix: - python-version: ["current", "previous", "next"] + # Full version spread on push (master/release) + nightly; current only + # on PRs to cut runner cost (cross-version breaks are caught at merge). + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "previous", "next"]') }} env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -129,9 +139,6 @@ jobs: services: postgres: image: postgres:17-alpine - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} env: POSTGRES_USER: superset POSTGRES_PASSWORD: superset @@ -141,40 +148,28 @@ jobs: - 15432:5432 redis: image: redis:7-alpine - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} ports: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Setup Postgres - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: | setup-postgres - name: Start Celery worker - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: celery-worker - name: Python integration tests (PostgreSQL) - if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage @@ -186,7 +181,10 @@ jobs: slug: apache/superset test-sqlite: + needs: changes + if: needs.changes.outputs.python == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 45 permissions: id-token: write env: @@ -200,39 +198,27 @@ jobs: services: redis: image: redis:7-alpine - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} ports: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python - name: Install dependencies - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: | # sqlite needs this working directory mkdir ${{ github.workspace }}/.temp - name: Start Celery worker - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: celery-worker - name: Python integration tests (SQLite) - if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage @@ -242,3 +228,25 @@ jobs: verbose: true use_oidc: true slug: apache/superset + + # Stable required-status-check anchor for the matrix-based test-postgres job. + # It is gated on change detection, so on non-Python PRs it is skipped and + # never produces its `test-postgres (current)` context (a job-level skip + # happens before matrix expansion). This always-running job reports a single + # context branch protection can require: it passes when test-postgres + # succeeded or was skipped, and fails only on a real failure. + test-postgres-required: + needs: [changes, test-postgres] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check test-postgres result + env: + RESULT: ${{ needs.test-postgres.result }} + run: | + if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then + echo "test-postgres did not pass (result: $RESULT)" + exit 1 + fi + echo "test-postgres result: $RESULT" diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index 264b124f8c5..260b144916c 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -15,8 +15,30 @@ concurrency: cancel-in-progress: true jobs: - test-postgres-presto: + changes: runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + + test-postgres-presto: + needs: changes + if: needs.changes.outputs.python == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 permissions: id-token: write env: @@ -50,32 +72,21 @@ jobs: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python == 'true' - name: Setup Postgres - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: - run: | - echo "${{ steps.check.outputs.python }}" - setup-postgres + run: setup-postgres - name: Start Celery worker - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: celery-worker - name: Python unit tests (PostgreSQL) - if: steps.check.outputs.python run: | ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow' - name: Upload code coverage @@ -87,7 +98,10 @@ jobs: slug: apache/superset test-postgres-hive: + needs: changes + if: needs.changes.outputs.python == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 45 permissions: id-token: write env: @@ -113,39 +127,27 @@ jobs: - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive - - name: Check for file changes - id: check - uses: ./.github/actions/change-detector/ - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Create csv upload directory - if: steps.check.outputs.python run: sudo mkdir -p /tmp/.superset/uploads - name: Give write access to the csv upload directory - if: steps.check.outputs.python run: sudo chown -R $USER:$USER /tmp/.superset - name: Start hadoop and hive - if: steps.check.outputs.python run: docker compose -f scripts/databases/hive/docker-compose.yml up -d - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python - name: Setup Postgres - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Start Celery worker - if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: celery-worker - name: Python unit tests (PostgreSQL) - if: steps.check.outputs.python run: | pip install -e .[hive] ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow' diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 5a71d311ac2..f30d8cf6f54 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -15,40 +15,56 @@ concurrency: cancel-in-progress: true jobs: - unit-tests: + changes: runs-on: ubuntu-24.04 + timeout-minutes: 10 permissions: - id-token: write - strategy: - matrix: - python-version: ["previous", "current", "next"] - env: - PYTHONPATH: ${{ github.workspace }} + contents: read + pull-requests: read + outputs: + python: ${{ steps.check.outputs.python }} steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - submodules: recursive - name: Check for file changes id: check uses: ./.github/actions/change-detector/ with: token: ${{ secrets.GITHUB_TOKEN }} + + unit-tests: + needs: changes + if: needs.changes.outputs.python == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + id-token: write + strategy: + matrix: + # Full version spread on push (master/release) + nightly; current only + # on PRs to cut runner cost (cross-version breaks are caught at merge). + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["previous", "current", "next"]') }} + env: + PYTHONPATH: ${{ github.workspace }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + submodules: recursive - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Python unit tests - if: steps.check.outputs.python env: SUPERSET_TESTENV: true SUPERSET_SECRET_KEY: not-a-secret run: | pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear --maxfail=50 - name: Python 100% coverage unit tests - if: steps.check.outputs.python env: SUPERSET_TESTENV: true SUPERSET_SECRET_KEY: not-a-secret @@ -62,3 +78,25 @@ jobs: verbose: true use_oidc: true slug: apache/superset + + # Stable required-status-check anchor. `unit-tests` is a matrix job gated on + # change detection, so on non-Python PRs it is skipped and never produces its + # `unit-tests (current)` context (a job-level skip happens before matrix + # expansion). This always-running job reports a single context that branch + # protection can require: it passes when unit-tests succeeded or was skipped, + # and fails only on a real failure. + unit-tests-required: + needs: [changes, unit-tests] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check unit-tests result + env: + RESULT: ${{ needs.unit-tests.result }} + run: | + if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then + echo "unit-tests did not pass (result: $RESULT)" + exit 1 + fi + echo "unit-tests result: $RESULT" diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index 784ef1457b5..c7aea034d49 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -25,7 +25,7 @@ jobs: pull-requests: read steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive @@ -40,7 +40,9 @@ jobs: if: steps.check.outputs.frontend uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" + cache: "npm" + cache-dependency-path: "superset-frontend/package-lock.json" - name: Install dependencies if: steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies @@ -59,7 +61,7 @@ jobs: pull-requests: read steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/superset-websocket.yml b/.github/workflows/superset-websocket.yml index 58e5ffe0552..f35d54dc013 100644 --- a/.github/workflows/superset-websocket.yml +++ b/.github/workflows/superset-websocket.yml @@ -22,9 +22,10 @@ concurrency: jobs: app-checks: runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Install dependencies diff --git a/.github/workflows/supersetbot.yml b/.github/workflows/supersetbot.yml index c187a45040e..dbca6df7841 100644 --- a/.github/workflows/supersetbot.yml +++ b/.github/workflows/supersetbot.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: comment_body: - description: 'Comment Body' + description: "Comment Body" required: true type: string @@ -38,7 +38,7 @@ jobs: }); - name: "Checkout ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index 370c8fa87a3..a5964d679be 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -16,11 +16,11 @@ on: force-latest: required: true type: choice - default: 'false' + default: "false" description: Whether to force a latest tag on the release options: - - 'true' - - 'false' + - "true" + - "false" permissions: contents: read @@ -49,12 +49,12 @@ jobs: contents: write strategy: matrix: - build_preset: ["dev", "lean", "py310", "websocket", "dockerize", "py311", "py312"] + build_preset: + ["dev", "lean", "py310", "websocket", "dockerize", "py311", "py312"] fail-fast: false steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false fetch-depth: 0 @@ -119,9 +119,8 @@ jobs: contents: read pull-requests: write steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/tech-debt.yml b/.github/workflows/tech-debt.yml index 41880594a9b..41d40f1dd94 100644 --- a/.github/workflows/tech-debt.yml +++ b/.github/workflows/tech-debt.yml @@ -32,14 +32,14 @@ jobs: name: Generate Reports steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version-file: './superset-frontend/.nvmrc' + node-version-file: "./superset-frontend/.nvmrc" - name: Install Dependencies run: npm ci diff --git a/Dockerfile b/Dockerfile index 6da35f323b2..ecdbb97c751 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ ARG BUILD_TRANSLATIONS="false" ###################################################################### # superset-node-ci used as a base for building frontend assets and CI ###################################################################### -FROM --platform=${BUILDPLATFORM} node:22-trixie-slim AS superset-node-ci +FROM --platform=${BUILDPLATFORM} node:24-trixie-slim AS superset-node-ci ARG BUILD_TRANSLATIONS ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} ARG DEV_MODE="false" # Skip frontend build in dev mode @@ -55,6 +55,13 @@ WORKDIR /app/superset-frontend RUN mkdir -p /app/superset/static/assets \ /app/superset/translations +# Harden `npm ci` against transient npm-registry network blips (e.g. ECONNRESET), +# which otherwise fail the entire multi-platform image build with no retry. +ENV npm_config_fetch_retries=5 \ + npm_config_fetch_retry_mintimeout=20000 \ + npm_config_fetch_retry_maxtimeout=120000 \ + npm_config_fetch_timeout=600000 + # Mount package files and install dependencies if not in dev mode # NOTE: we mount packages and plugins as they are referenced in package.json as workspaces # ideally we'd COPY only their package.json. Here npm ci will be cached as long diff --git a/README.md b/README.md index e40a0b572b9..d328add67ca 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,11 @@ Try out Superset's [quickstart](https://superset.apache.org/docs/quickstart/) gu - [Join our community's Slack](http://bit.ly/join-superset-slack) and please read our [Slack Community Guidelines](https://github.com/apache/superset/blob/master/CODE_OF_CONDUCT.md#slack-community-guidelines) - [Join our dev@superset.apache.org Mailing list](https://lists.apache.org/list.html?dev@superset.apache.org). To join, simply send an email to [dev-subscribe@superset.apache.org](mailto:dev-subscribe@superset.apache.org) +- Follow us on social media: + [X](https://x.com/apachesuperset) | + [LinkedIn](https://www.linkedin.com/company/apache-superset) | + [Bluesky](https://bsky.app/profile/apachesuperset.bsky.social) | + [Reddit](https://reddit.com/r/apache-superset) - If you want to help troubleshoot GitHub Issues involving the numerous database drivers that Superset supports, please consider adding your name and the databases you have access to on the [Superset Database Familiarity Rolodex](https://docs.google.com/spreadsheets/d/1U1qxiLvOX0kBTUGME1AHHi6Ywel6ECF8xk_Qy-V9R8c/edit#gid=0) - Join Superset's Town Hall and [Operational Model](https://preset.io/blog/the-superset-operational-model-wants-you/) recurring meetings. Meeting info is available on the [Superset Community Calendar](https://superset.apache.org/community) diff --git a/UPDATING.md b/UPDATING.md index e1012be14d1..d0208eb5894 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,37 @@ assists people when migrating to a new version. ## Next +### Map chart renderer and OpenStreetMap migration behavior + +The MapLibre migration for deck.gl charts preserves saved non-Mapbox styles on +the MapLibre-compatible path. Saved styles such as OpenStreetMap, `tile://` +tile templates, generic HTTPS style URLs, and charts without a saved style are +not reclassified as Mapbox during migration and do not require +`MAPBOX_API_KEY` only because of the migration. + +Saved true Mapbox styles whose value starts with `mapbox://` remain +Mapbox-backed. If a Superset deployment does not configure `MAPBOX_API_KEY`, +those saved Mapbox charts keep the existing missing-key message instead of +silently falling back to MapLibre or another provider. In Explore, deck.gl and +point-cluster renderer controls preserve saved Mapbox state, but the Mapbox +choice is not available as a new working renderer without a configured key. + +The MapLibre style choices include `Streets (OSM)`, backed by +`https://tile.openstreetmap.org/{z}/{x}/{y}.png`. This OpenStreetMap tile +service requires visible `© OpenStreetMap contributors` attribution and should +be used through normal browser map tile requests and caching; it is not intended +for bulk prefetch or offline tile downloads. + +### Duration formatter precision + +The `DURATION` number formatter now uses `Intl.DurationFormat` for locale-aware output. By default, sub-second fields are omitted, so values that previously displayed fractional seconds with `pretty-ms`, such as `10500` milliseconds rendering as `10.5s`, now render as `10s`. + +To preserve sub-second precision in custom duration formatters, enable `formatSubMilliseconds`. + +### Cache warmup authenticates via SUPERSET_CACHE_WARMUP_USER + +The `cache-warmup` Celery task now drives a real WebDriver session for reliable authentication and reads the user to authenticate as from the new `SUPERSET_CACHE_WARMUP_USER` config option. It no longer consults `CACHE_WARMUP_EXECUTORS` for the warmup path. `SUPERSET_CACHE_WARMUP_USER` defaults to `None`, so the task fails fast with a clear message until you set it. Operators who previously relied on `CACHE_WARMUP_EXECUTORS` for cache warmup must set `SUPERSET_CACHE_WARMUP_USER` to a dedicated least-privilege user with access to the dashboards they want warmed up before the next warmup run. + ### YDB now uses a native sqlglot dialect YDB SQL parsing now relies on the dedicated [`ydb-sqlglot-plugin`](https://pypi.org/project/ydb-sqlglot-plugin/) dialect, which registers itself with sqlglot automatically. YDB users must install this plugin (e.g., via `pip install "apache-superset[ydb]"`) to avoid a `ValueError` when Superset parses YDB queries. @@ -34,12 +65,51 @@ The embedded dashboard page now validates the origin of incoming `postMessage` e Enforcement only applies when the Allowed Domains list is non-empty. If the list is empty (the default), any origin is accepted, so there is no behavior change for embeds that did not configure Allowed Domains. +### Default guest/async JWT secrets are rejected at startup + +Superset already refuses to start in production (non-debug, non-testing) when `SECRET_KEY` is left at its built-in default, and when `GUEST_TOKEN_JWT_SECRET` is left at its default while `EMBEDDED_SUPERSET` is enabled. This behavior is extended to `GLOBAL_ASYNC_QUERIES_JWT_SECRET`: if the `GLOBAL_ASYNC_QUERIES` feature flag is enabled and the secret is still the publicly known default (`test-secret-change-me`), Superset logs a clear error and refuses to start. + +As with the existing `SECRET_KEY` check, this only fails in production. In debug mode, testing mode, or under the test runner, a warning is logged instead of exiting, so local development is unaffected. + +To resolve the error, set a strong random value in `superset_config.py`: + +```python +GLOBAL_ASYNC_QUERIES_JWT_SECRET = "" +``` + +The check is only active when the relevant feature is enabled, so deployments that do not use global async queries (or embedding) are not affected. + +### Guest token revocation (opt-in) + +Embedded guest tokens can be coarsely revoked at runtime via a new opt-in mechanism. A new config flag `GUEST_TOKEN_REVOCATION_ENABLED` (default `False`) gates the feature. When enabled, every minted guest token carries a revocation version, and tokens whose version is below the current expected version (stored in the metadata database) are rejected at validation time. + +Bump the expected version with the new CLI command to invalidate all outstanding guest tokens: + +```bash +superset revoke-guest-tokens +``` + +This change is backward compatible. The feature is off by default, and even when enabled nothing is revoked until an admin explicitly bumps the version: the expected version starts at `0`, and tokens minted before this change (which carry no version claim) are treated as version `0`. No database migration is required. + ### Dataset import validates catalog against the target connection Importing a dataset now validates the `catalog` field against the target database connection. When the connection has multi-catalog disabled (`allow_multi_catalog` off) and the dataset's catalog is not the connection's default catalog, the import fails instead of silently persisting the non-default catalog. This matches the validation already enforced on the dataset update path and prevents imported datasets from querying an unintended database. If you relied on importing datasets with a non-default catalog, enable "Allow changing catalogs" on the target connection, or set the dataset's catalog to the connection's default before importing. +### Extension supply-chain controls (denylist + version policy) + +Two opt-in static gates control which extensions are allowed to load: + +- `EXTENSION_DENYLIST` refuses extensions matching an id (every version) or `id@version` (a single version), e.g. `["compromised-extension", "other-ext@1.2.3"]`. +- `EXTENSION_VERSION_POLICY` enforces a minimum version per extension id, e.g. `{"acme.widget": "1.2.0"}` (PEP 440 comparison); a release below the minimum is refused. + +Both default to empty (no behavior change). They apply to both the `LOCAL_EXTENSIONS` and `EXTENSIONS_PATH` load paths. + +### Dynamic Group By respects the sort toggle for display values + +The Dynamic Group By chart customization now orders its display values according to the "Sort display control values" toggle: ascending (A–Z), descending (Z–A), or the dataset's source order when the toggle is unset. Previously the dropdown always sorted alphabetically. Existing dashboards where the toggle was never set will show options in source order instead of A–Z; open the customization and enable the toggle to restore alphabetical ordering. + ### Granular Export Controls A new feature flag `GRANULAR_EXPORT_CONTROLS` introduces three fine-grained permissions that replace the legacy `can_csv` permission: diff --git a/docker-compose.yml b/docker-compose.yml index bd474a83ef4..6d1e2bc3f05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,31 @@ services: volumes: - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./docker/nginx/templates:/etc/nginx/templates:ro + # Wait for the webpack dev server's manifest.json to be served before + # starting nginx. This prevents 404s on static assets at startup. The + # probe targets host.docker.internal so it works regardless of whether + # the dev server runs in the superset-node container + # (BUILD_SUPERSET_FRONTEND_IN_DOCKER=true, the default) or directly on + # the host (BUILD_SUPERSET_FRONTEND_IN_DOCKER=false). + command: + - /bin/bash + - -c + - | + url="http://host.docker.internal:9000/static/assets/manifest.json" + max_attempts=150 # ~5 minutes at 2s intervals + echo "Waiting for webpack dev server at $url..." + attempt=0 + until curl -sf --max-time 5 -o /dev/null "$url"; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge "$max_attempts" ]; then + echo "ERROR: webpack dev server did not serve $url after $max_attempts attempts (~5 minutes)." >&2 + echo "Is the dev server running? With BUILD_SUPERSET_FRONTEND_IN_DOCKER=false you must start it on the host (e.g. 'npm run dev' in superset-frontend)." >&2 + exit 1 + fi + sleep 2 + done + echo "Webpack dev server is ready; starting nginx." + exec nginx -g 'daemon off;' redis: image: redis:7 diff --git a/docs/.nvmrc b/docs/.nvmrc index 42a1c98ac5a..eb6ead3cf55 100644 --- a/docs/.nvmrc +++ b/docs/.nvmrc @@ -1 +1 @@ -v22.22.0 +v24.16.0 diff --git a/docs/admin_docs/configuration/cache.mdx b/docs/admin_docs/configuration/cache.mdx index ef3fbe1161c..b7aeff8d7aa 100644 --- a/docs/admin_docs/configuration/cache.mdx +++ b/docs/admin_docs/configuration/cache.mdx @@ -86,6 +86,39 @@ instead requires a cachelib object. See [Async Queries via Celery](/admin-docs/configuration/async-queries-celery) for details. +## Celery beat + +Superset has a Celery task that will periodically warm up the cache based on different strategies. +To use it, add the following to your `superset_config.py`: + +```python +from celery.schedules import crontab +from superset.config import CeleryConfig + +# User that will be used to authenticate and render dashboards for cache warmup +SUPERSET_CACHE_WARMUP_USER = "user_with_permission_to_dashboards" + +# Extend the default CeleryConfig to add cache warmup schedule +class CustomCeleryConfig(CeleryConfig): + beat_schedule = { + **CeleryConfig.beat_schedule, + 'cache-warmup-hourly': { + 'task': 'cache-warmup', + 'schedule': crontab(minute=0, hour='*'), # hourly + 'kwargs': { + 'strategy_name': 'top_n_dashboards', + 'top_n': 5, + 'since': '7 days ago', + }, + }, + } + +CELERY_CONFIG = CustomCeleryConfig +``` + +This will cache the top 5 most popular dashboards every hour. For other +strategies, check the `superset/tasks/cache.py` file. + ## Caching Thumbnails This is an optional feature that can be turned on by activating its [feature flag](/admin-docs/configuration/configuring-superset#feature-flags) on config: diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 8c226188f6d..7c7f920187b 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -917,6 +917,23 @@ const config: Config = { footer: { links: [], copyright: ` +