[Build] Add Github workflows (#9517)

As an replacement to Travis CI.

.github/workflows/bashlib.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Echo only when not in parallel mode
+say() {
+  if [[ ${INPUT_PARALLEL^^} != 'TRUE' ]]; then
+    echo "$1"
+  fi
+}
+
+# default command to run when the `run` input is empty
+default-setup-command() {
+  pip-install
+}
+
+# install python dependencies
+pip-install() {
+  cd "$GITHUB_WORKSPACE"
+
+  # Don't use pip cache as it doesn't seem to help much.
+  # cache-restore pip
+
+  say "::group::Install Python pacakges"
+  pip install -r requirements.txt
+  pip install -r requirements-dev.txt
+  pip install -e ".[postgres,mysql]"
+  say "::endgroup::"
+
+  # cache-save pip
+}
+
+# prepare (lint and build) frontend code
+npm-install() {
+  cd "$GITHUB_WORKSPACE/superset-frontend"
+
+  cache-restore npm
+
+  say "::group::Install npm packages"
+  echo "npm: $(npm --version)"
+  echo "node: $(node --version)"
+  npm ci
+  say "::endgroup::"
+
+  cache-save npm
+}
+
+build-assets() {
+  cd "$GITHUB_WORKSPACE/superset-frontend"
+
+  say "::group::Build static assets"
+  npm run build -- --no-progress
+  say "::endgroup::"
+}
+
+npm-build() {
+  if [[ $1 = '--no-cache' ]]; then
+    build-assets
+  else
+    cache-restore assets
+    if [[ -f $GITHUB_WORKSPACE/superset/static/assets/manifest.json ]]; then
+      echo 'Skip frontend build because static assets already exist.'
+    else
+      build-assets
+      cache-save assets
+    fi
+  fi
+}
+
+cypress-install() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+
+  cache-restore cypress
+
+  say "::group::Install Cypress"
+  npm ci
+  say "::endgroup::"
+
+  cache-save cypress
+}
+
+testdata() {
+  cd "$GITHUB_WORKSPACE"
+  say "::group::Load test data"
+  # must specify PYTHONPATH to make `tests.superset_test_config` importable
+  export PYTHONPATH="$GITHUB_WORKSPACE"
+  superset db upgrade
+  superset load_test_users
+  superset load_examples --load-test-data
+  superset init
+  say "::endgroup::"
+}
+
+setup-postgres() {
+  say "::group::Initialize database"
+  psql "postgresql://superset:superset@127.0.0.1:15432/superset" <<-EOF
+    DROP SCHEMA IF EXISTS sqllab_test_db;
+    CREATE SCHEMA sqllab_test_db;
+    DROP SCHEMA IF EXISTS admin_database;
+    CREATE SCHEMA admin_database;
+EOF
+  say "::endgroup::"
+}
+
+setup-mysql() {
+  say "::group::Initialize database"
+  mysql -h 127.0.0.1 -P 13306 -u root --password=root <<-EOF
+    DROP DATABASE IF EXISTS superset;
+    CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+    DROP DATABASE IF EXISTS sqllab_test_db;
+    CREATE DATABASE sqllab_test_db DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+    DROP DATABASE IF EXISTS admin_database;
+    CREATE DATABASE admin_database DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+    CREATE USER 'superset'@'%' IDENTIFIED BY 'superset';
+    GRANT ALL ON *.* TO 'superset'@'%';
+    FLUSH PRIVILEGES;
+EOF
+  say "::endgroup::"
+}

.github/workflows/caches.js

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// always use absolute directory
+const workspaceDirectory = process.env.GITHUB_WORKSPACE;
+const homeDirectory = process.env.HOME;
+
+// Multi-layer cache definition
+module.exports = {
+  pip: {
+    path: [`${homeDirectory}/.cache/pip`],
+    hashFiles: [`${workspaceDirectory}/requirements*.txt`],
+  },
+  npm: {
+    path: [`${homeDirectory}/.npm`],
+    hashFiles: ['superset-frontend/package-lock.json'],
+  },
+  assets: {
+    path: [
+      `${workspaceDirectory}/superset/static/assets`,
+    ],
+    hashFiles: [
+      `${workspaceDirectory}/superset-frontend/src/**/*`,
+      `${workspaceDirectory}/superset-frontend/*.json`,
+      `${workspaceDirectory}/superset-frontend/*.js`,
+    ],
+    // dont use restore keys as it may give an invalid older build
+    restoreKeys: ''
+  },
+  cypress: {
+    path: [`${homeDirectory}/.cache/Cypress`],
+    hashFiles: [
+      `${workspaceDirectory}/superset-frontend/cypress-base/package-lock.json`,
+    ],
+  },
+};

.github/workflows/license-check.yml

@@ -0,0 +1,22 @@
+name: License
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  check:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Generate fossa report
+      env:
+        FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
+      run: ./scripts/fossa.sh
+    - name: Run license check
+      run: ./scripts/check_license.sh

.github/workflows/superset-e2e.yml

@@ -0,0 +1,91 @@
+name: E2E
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  cypress:
+    name: Cypress
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: ['chrome']
+    env:
+      FLASK_ENV: development
+      SUPERSET_CONFIG: tests.superset_test_config
+      SUPERSET__SQLALCHEMY_DATABASE_URI:
+        postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      PYTHONPATH: ${{ github.workspace }}
+      REDIS_PORT: 16379
+      CI: github-actions
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+    services:
+      postgres:
+        image: postgres:10-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+      with:
+        # Run commands in parallel does help initial installation without cache
+        parallel: true
+        run: |
+          npm-install && npm-build
+          pip-install && setup-postgres && testdata
+          cypress-install
+
+    - name: Cypress run all
+      env:
+        CYPRESS_GROUP: Default
+        CYPRESS_PATH: 'cypress/integration/*/*'
+      run: |
+        # Start Flask and run Cypress
+
+        # --no-debugger means disable the interactive debugger on the 500 page
+        # so errors can print to stderr.
+        flask run --no-debugger --with-threads -p 8081 &
+
+        sleep 3 # wait for the Flask app to start
+
+        cd ${{ github.workspace }}/superset-frontend/cypress-base/
+        npm run cypress -- run \
+          --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" \
+          --record --group "${{ env.CYPRESS_GROUP }}" \
+          --ci-build-id ${{ github.event_name }}-${{ github.run_id }}
+
+    - name: Cypress run SQL Lab (with backend persist)
+      env:
+        SUPERSET_CONFIG: tests.superset_test_config_sqllab_backend_persist
+        CYPRESS_GROUP: Backend persist
+        CYPRESS_PATH: 'cypress/integration/sqllab/*'
+      run: |
+        # Start Flask with alternative config and run Cypress
+
+        killall python # exit the running Flask app
+        flask run --no-debugger --with-threads -p 8081 &
+        sleep 3 # wait for the Flask app to start
+
+        cd ${{ github.workspace }}/superset-frontend/cypress-base/
+        npm run cypress -- run \
+          --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" \
+          --record --group "${{ env.CYPRESS_GROUP }}" \
+          --ci-build-id ${{ github.event_name }}-${{ github.run_id }}

.github/workflows/superset-frontend.yml

@@ -0,0 +1,36 @@
+name: Frontend
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - superset-frontend/**
+  pull_request:
+    paths:
+    - superset-frontend/**
+
+jobs:
+  frontend-build:
+    name: build
+    runs-on: ubuntu-18.04
+    env:
+      CI: github-actions
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+      with:
+        run: npm-install
+    - name: eslint
+      working-directory: ./superset-frontend
+      run: |
+        npm run lint
+    - name: unit tests
+      working-directory: ./superset-frontend
+      run: |
+        npm run test -- --coverage
+    - name: Upload code coverage
+      working-directory: ./superset-frontend
+      run: |
+        bash <(curl -s https://codecov.io/bash) -cF unittest,javascript

.github/workflows/superset-python.yml

@@ -0,0 +1,174 @@
+name: Python
+
+on:
+  # only build on direct push to `master` branch
+  push:
+    branches: [ master ]
+    paths:
+    - ./**/*.py
+    - superset/**
+    - tests/**
+    - requirements*.txt
+  # but also build on pull requests to any branch
+  # (the so-called feature branch)
+  pull_request:
+    paths:
+    - ./**/*.py
+    - superset/**
+    - tests/**
+    - setup.py
+    - requirements*.txt
+
+jobs:
+  lint:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.6]
+    env:
+      PYTHON_LINT_TARGET: setup.py superset tests
+      CI: github-actions
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+    - name: black
+      run: black --check $(echo $PYTHON_LINT_TARGET)
+    - name: mypy
+      run: mypy $(echo $PYTHON_LINT_TARGET)
+    - name: isort
+      run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET)
+    - name: pylint
+      # `-j 0` run Pylint in parallel
+      run: pylint -j 0 superset
+
+  test-postgres:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        # run unit tests in multiple version just for fun
+        # (3.8 is not supported yet, some dependencies need an update)
+        python-version: [3.6, 3.7]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.superset_test_config
+      REDIS_PORT: 16379
+    services:
+      postgres:
+        image: postgres:10-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          # Use custom ports for services to avoid accidentally connecting to
+          # GitHub action runner's default installations
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+      with:
+        run: |
+          pip-install
+          setup-postgres
+    - name: Python unit tests (PostgreSQL)
+      env:
+        SUPERSET__SQLALCHEMY_DATABASE_URI:
+          postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      run: |
+        ./scripts/python_tests.sh
+    - name: Upload code coverage
+      run: |
+        bash <(curl -s https://codecov.io/bash) -cF unittest,python,postgres
+
+  test-mysql:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.6]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.superset_test_config
+      REDIS_PORT: 16379
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 13306:3306
+      redis:
+        image: redis:5-alpine
+        options: --entrypoint redis-server
+        ports:
+          - 16379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+      with:
+        run: |
+          pip-install
+          setup-mysql
+    - name: Python unit tests (MySQL)
+      env:
+        SUPERSET__SQLALCHEMY_DATABASE_URI: |
+          mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
+      run: |
+        ./scripts/python_tests.sh
+    - name: Upload code coverage
+      run: |
+        bash <(curl -s https://codecov.io/bash) -cF unittest,python,mysql
+
+  test-sqlite:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.6]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.superset_test_config
+      REDIS_PORT: 16379
+    services:
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@ddf7d7f
+      with:
+        run: |
+          pip-install
+          mkdir ${{ github.workspace }}/.temp
+    - name: Python unit tests (SQLite)
+      env:
+        SUPERSET__SQLALCHEMY_DATABASE_URI: |
+          sqlite:///${{ github.workspace }}/.temp/unittest.db
+      run: |
+        ./scripts/python_tests.sh
+    - name: Upload code coverage
+      run: |
+        bash <(curl -s https://codecov.io/bash) -cF unittest,python,sqlite