QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-24 02:25:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

build: try to speed up Github workflows (#12090)

Browse Source
This commit is contained in:
Jesse Yang
2021-01-07 00:47:00 -08:00
committed by GitHub
parent bcb8db621c
commit f48284909d
15 changed files with 503 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/cancel_duplicates.yml vendored Normal file
View File

@@ -0,0 +1,20 @@
name: Cancel Duplicates
on:
workflow_run:
workflows: ["CI"]
types: ["requested"]
jobs:
cancel-duplicate-workflow-runs:
name: "Cancel duplicate workflow runs"
runs-on: ubuntu-latest
steps:
- uses: apache-superset/cancel-workflow-runs@953e057
name: "Cancel duplicate workflow runs"
with:
cancelMode: duplicates
cancelFutureDuplicates: true
token: ${{ secrets.GITHUB_TOKEN }}
sourceRunId: ${{ github.event.workflow_run.id }}
notifyPRCancel: true
skipEventTypes: '["push", "pull_request", "pull_request_target"]'

45
.github/workflows/license-check.yml vendored
View File

@@ -1,45 +0,0 @@
name: License
on:
push:
branches: [ master ]
pull_request:
jobs:
check:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 8
- name: Generate fossa report
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
run: |
set -eo pipefail
if [[ "${{github.event_name}}" != "pull_request" ]]; then
./scripts/fossa.sh
exit 0
fi
URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
FILES=$(curl -s -X GET -G $URL | jq -r '.[] | .filename')
cat<<EOF
CHANGED FILES:
$FILES
EOF
if [[ "${FILES}" =~ (.*package*\.json|requirements\/[a-z_-]+\.txt|setup\.py) ]]; then
echo "Detected dependency changes... running fossa check"
./scripts/fossa.sh
else
echo "No dependency changes... skiping fossa check"
fi
shell: bash
- name: Run license check
run: ./scripts/check_license.sh

91
.github/workflows/misc.yml vendored Normal file
View File

@@ -0,0 +1,91 @@
name: Miscellaneous
on:
push:
branches-ignore:
- "dependabot/**"
pull_request:
jobs:
license_check:
name: License Check
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 8
- name: Generate fossa report
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
run: |
set -eo pipefail
if [[ "${{github.event_name}}" != "pull_request" ]]; then
./scripts/fossa.sh
exit 0
fi
URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
FILES=$(curl -s -X GET -G $URL | jq -r '.[] | .filename')
cat<<EOF
CHANGED FILES:
$FILES
EOF
if [[ "${FILES}" =~ (.*package*\.json|requirements\/[a-z_-]+\.txt|setup\.py) ]]; then
echo "Detected dependency changes... running fossa check"
./scripts/fossa.sh
else
echo "No dependency changes... skiping fossa check"
fi
shell: bash
- name: Run license check
run: ./scripts/check_license.sh
prefer_typescript:
if: github.ref == 'ref/heads/master' && github.event_name == 'pull_request'
name: Prefer Typescript
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get changed files
id: changed
uses: ./.github/actions/file-changes-action
with:
githubToken: ${{ secrets.GITHUB_TOKEN }}
- name: Determine if a .js or .jsx file was added
id: check
run: |
js_files_added() {
jq -r '
map(
select(
endswith(".js") or endswith(".jsx")
)
) | join("\n")
' ${HOME}/files_added.json
}
echo ::set-output name=js_files_added::$(js_files_added)
- if: steps.check.outputs.js_files_added
name: Add Comment to PR
uses: ./.github/actions/comment-on-pr
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
msg: |
### WARNING: Prefer TypeScript
Looks like your PR contains new `.js` or `.jsx` files:
```
${{steps.check.outputs.js_files_added}}
```
As decided in [SIP-36](https://github.com/apache/superset/issues/9101), all new frontend code should be written in TypeScript. Please convert above files to TypeScript then re-request review.

49
.github/workflows/prefer-typescript.yml vendored
View File

@@ -1,49 +0,0 @@
name: Prefer TypeScript
on:
pull_request:
branches:
- master
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get changed files
id: changed
uses: ./.github/actions/file-changes-action
with:
githubToken: ${{ secrets.GITHUB_TOKEN }}
- name: Determine if a .js or .jsx file was added
id: check
run: |
js_files_added() {
jq -r '
map(
select(
endswith(".js") or endswith(".jsx")
)
) | join("\n")
' ${HOME}/files_added.json
}
echo ::set-output name=js_files_added::$(js_files_added)
- if: steps.check.outputs.js_files_added
name: Add Comment to PR
uses: ./.github/actions/comment-on-pr
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
msg: |
### WARNING: Prefer TypeScript
Looks like your PR contains new `.js` or `.jsx` files:
```
${{steps.check.outputs.js_files_added}}
```
As decided in [SIP-36](https://github.com/apache/superset/issues/9101), all new frontend code should be written in TypeScript. Please convert above files to TypeScript then re-request review.

8
.github/workflows/superset-e2e.yml vendored
View File

@@ -1,6 +1,12 @@
name: E2E
on: [push, pull_request]
on:
push:
branches-ignore:
- "dependabot/**/docs/**"
paths-ignore:
- "docs/**"
pull_request:
jobs:
Cypress:

7
.github/workflows/superset-frontend.yml vendored
View File

@@ -1,6 +1,11 @@
name: Frontend
on: [push, pull_request]
on:
push:
branches-ignore:
- "dependabot/**/docs/**"
- "dependabot/**/cypress-base/**"
pull_request:
jobs:
build:

6
.github/workflows/superset-python-misc.yml vendored
View File

@@ -1,7 +1,11 @@
# Python Misc unit tests
name: Python Misc
on: [push, pull_request]
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
jobs:
lint:

51
.github/workflows/superset-python-mysql.yml vendored
View File

@@ -1,51 +0,0 @@
# Python MySQL unit tests
name: Python MySQL
on: [push, pull_request]
jobs:
test-mysql:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
ports:
- 13306:3306
redis:
image: redis:5-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

52
.github/workflows/superset-python-postgres.yml vendored
View File

@@ -1,52 +0,0 @@
# Python Postgres unit tests
name: Python Postgres
on: [push, pull_request]
jobs:
test-postgres:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

6
.github/workflows/superset-python-presto-hive.yml vendored
View File

@@ -1,7 +1,11 @@
# Python Presto/Hive unit tests
name: Python Presto/Hive
on: [push, pull_request]
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
jobs:
test-postgres-presto:

44
.github/workflows/superset-python-sqlite.yml vendored
View File

@@ -1,44 +0,0 @@
# Python Sqlite unit tests
name: Python Sqlite
on: [push, pull_request]
jobs:
test-sqlite:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

141
.github/workflows/superset-python-unittest.yml vendored Normal file
View File

@@ -0,0 +1,141 @@
# Python MySQL unit tests
name: Python MySQL
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
jobs:
test-mysql:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
ports:
- 13306:3306
redis:
image: redis:5-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-postgres:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-sqlite:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

6
.github/workflows/superset-translations.yml vendored
View File

@@ -1,6 +1,10 @@
name: Translations
on: [push, pull_request]
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
jobs:
frontend-check: