QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 08:04:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

build: split python CI jobs (#11794)

Browse Source 
* build: split python CI jobs

* Rename superset-python.yml to superset-python-misc.yml
This commit is contained in:
Erik Ritter
2020-11-24 11:40:16 -08:00
committed by GitHub
parent 6019113bc6
commit 67c44479b5
6 changed files with 338 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
.github/workflows/superset-python-misc.yml vendored Normal file
View File

@@ -0,0 +1,72 @@
# Python Misc unit tests
name: Python Misc
on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
- name: pylint
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
pre-commit:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/integration.txt
- name: pre-commit
run: pre-commit run --all-files
babel-extract:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/base.txt
- name: Test babel extraction
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

51
.github/workflows/superset-python-mysql.yml vendored Normal file
View File

@@ -0,0 +1,51 @@
# Python MySQL unit tests
name: Python MySQL
on: [push, pull_request]
jobs:
test-mysql:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
ports:
- 13306:3306
redis:
image: redis:5-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

52
.github/workflows/superset-python-postgres.yml vendored Normal file
View File

@@ -0,0 +1,52 @@
# Python Postgres unit tests
name: Python Postgres
on: [push, pull_request]
jobs:
test-postgres:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

119
.github/workflows/superset-python-presto-hive.yml vendored Normal file
View File

@@ -0,0 +1,119 @@
# Python Presto/Hive unit tests
name: Python Presto/Hive
on: [push, pull_request]
jobs:
test-postgres-presto:
runs-on: ubuntu-18.04
strategy:
matrix:
# run unit tests in multiple version just for fun
python-version: [3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
presto:
image: prestosql/presto:339
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15433:8080
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-postgres-hive:
runs-on: ubuntu-18.04
strategy:
matrix:
# run unit tests in multiple version just for fun
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Create csv upload directory
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

44
.github/workflows/superset-python-sqlite.yml vendored Normal file
View File

@@ -0,0 +1,44 @@
# Python Sqlite unit tests
name: Python Sqlite
on: [push, pull_request]
jobs:
test-sqlite:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python

322
.github/workflows/superset-python.yml vendored
View File

@@ -1,322 +0,0 @@
# Python unit tests
name: Python
on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
- name: pylint
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
pre-commit:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/integration.txt
- name: pre-commit
run: pre-commit run --all-files
babel-extract:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/base.txt
- name: Test babel extraction
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
test-postgres-presto:
runs-on: ubuntu-18.04
strategy:
matrix:
# run unit tests in multiple version just for fun
python-version: [3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI:
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI:
presto://localhost:15433/memory/default
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
presto:
image: prestosql/presto:339
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15433:8080
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-postgres-hive:
runs-on: ubuntu-18.04
strategy:
matrix:
# run unit tests in multiple version just for fun
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI:
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Create csv upload directory
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-postgres:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7, 3.8]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI:
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:10-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-mysql:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
ports:
- 13306:3306
redis:
image: redis:5-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python
test-sqlite:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:5-alpine
ports:
- 16379:6379
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
run: |
./scripts/python_tests.sh
- name: Upload code coverage
run: |
bash <(curl -s https://codecov.io/bash) -cF python