fix: improve change detection for GHAs (#27904)

This commit is contained in:
Maxime Beauchemin
2024-04-08 16:20:25 -07:00
committed by GitHub
parent 7c8e1bb46e
commit e80d194b8f
16 changed files with 338 additions and 331 deletions

View File

@@ -3,13 +3,9 @@ name: "CodeQL"
on:
push:
branches: ["master", "[0-9].[0-9]"]
paths:
- "superset/**"
pull_request:
# The branches below must be a subset of the branches above
branches: ["master"]
paths:
- "superset/**"
schedule:
- cron: "0 4 * * *"
@@ -37,6 +33,12 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
@@ -50,6 +52,7 @@ jobs:
# queries: security-extended,security-and-quality
- name: Perform CodeQL Analysis
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"

View File

@@ -39,28 +39,39 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: docker/setup-buildx-action@v3
- name: Try to login to DockerHub
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup supersetbot
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: ./.github/actions/setup-supersetbot/
- name: Build Docker Image
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
shell: bash
run: |
# Single platform builds in pull_request context to speed things up

View File

@@ -1,114 +0,0 @@
# no-op.yml
#
# Purpose:
# This workflow provides a workaround for the "required status checks" feature in GitHub Actions
# when using path-specific conditions in other workflows. Required checks might remain in a "Pending"
# state if the conditions are not met, thus blocking pull requests from being merged.
# This no-op (no operation) workflow provides dummy success statuses for these required jobs when
# the real jobs do not run due to path-specific conditions.
#
# How it works:
# - It defines jobs with the same names as the required jobs in the main workflows.
# - These jobs simply execute a command (`exit 0`) to succeed immediately.
# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered.
# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses.
# - If the main workflows' jobs do run and fail, their failure statuses take precedence,
# ensuring that pull requests are not merged with failing checks.
#
# Usage:
# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows.
# - This workflow should be kept as-is, without path-specific conditions.
name: no-op Checks
on: pull_request
jobs:
frontend-build:
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for frontend-build to ensure a successful status."
exit 0
pre-commit:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for pre-commit
run: |
echo "This is a no-op step for pre-commit to ensure a successful status."
exit 0
python-lint:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for python-lint
run: |
echo "This is a no-op step for python-lint to ensure a successful status."
exit 0
test-postgres-hive:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
exit 0
test-postgres-presto:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
exit 0
unit-tests:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for unit-tests to ensure a successful status when skipped."
exit 0
test-mysql:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-mysql
run: |
echo "This is a no-op step for test-mysql to ensure a successful status when skipped."
exit 0
test-postgres:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-postgres
run: |
echo "This is a no-op step for test-postgres to ensure a successful status when skipped."
exit 0
test-sqlite:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-sqlite
run: |
echo "This is a no-op step for test-sqlite to ensure a successful status when skipped."
exit 0

View File

@@ -44,29 +44,27 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: superset init
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
pip install -e .
superset db upgrade
superset load_test_users
- name: superset load_examples
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
# load examples without test data
superset load_examples --load-big-data

View File

@@ -60,47 +60,46 @@ jobs:
ref: "refs/pull/${{ github.event.number }}/merge"
persist-credentials: false
submodules: recursive
- name: Check if python or frontend changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python frontend
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
- name: Setup postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Import test data
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: testdata
- name: Setup Node.js
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "16"
- name: Install npm dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Build javascript packages
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: build-instrumented-assets
- name: Install cypress
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: cypress-install
- name: Run Cypress
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
@@ -109,7 +108,7 @@ jobs:
run: cypress-run-all
- name: Upload Artifacts
uses: actions/upload-artifact@v4
if: failure()
if: steps.check.outputs.python || steps.check.outputs.frontend
with:
name: screenshots
path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots

View File

@@ -5,12 +5,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset-frontend/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset-frontend/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -28,62 +24,60 @@ jobs:
submodules: recursive
- name: Check npm lock file version
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
- name: Check if frontend changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh frontend
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Node.js
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "18"
- name: Install dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: eslint
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run eslint -- . --quiet
- name: tsc
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run type
- name: prettier
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run prettier-check
- name: Build plugins packages
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Build plugins Storybook
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build-storybook
- name: superset-ui/core coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run core:cover
- name: unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run test -- --coverage --silent
# todo: remove this step when fix generator as a project in root jest.config.js
- name: generator-superset unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend/packages/generator-superset
run: npx jest
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: ../.github/workflows/codecov.sh -c -F javascript

View File

@@ -43,33 +43,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup MySQL
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (MySQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F mysql
test-postgres:
@@ -102,33 +100,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F postgres
@@ -156,33 +152,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
# sqlite needs this working directory
mkdir ${{ github.workspace }}/.temp
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (SQLite)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F sqlite

View File

@@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -30,20 +26,18 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: pylint
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
@@ -58,9 +52,16 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
if: steps.check.outputs.python
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

View File

@@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -59,30 +55,30 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python == 'true'
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
run: |
echo "${{ steps.check.outputs.python }}"
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F presto
@@ -118,38 +114,36 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F hive

View File

@@ -6,18 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
- "requirements/**"
- "tests/unit_tests/**"
- "scripts/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
- "requirements/**"
- "tests/unit_tests/**"
- "scripts/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -38,26 +28,24 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Python unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
env:
SUPERSET_TESTENV: true
SUPERSET_SECRET_KEY: not-a-secret
run: |
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F unit

View File

@@ -14,7 +14,7 @@ concurrency:
cancel-in-progress: true
jobs:
frontend-check:
frontend-check-translations:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -22,15 +22,25 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Node.js
if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Install dependencies
if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run check-translation
@@ -46,9 +56,17 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
if: steps.check.outputs.python
run: ./scripts/babel_update.sh