build: optimize GitHub Actions for building speed and stability (#10485)

.github/workflows/bashlib.sh

@@ -20,6 +20,10 @@ set -e
 GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
 ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
 
+# Rounded job start time, used to create a unique Cypress build id for
+# parallelization so we can manually rerun a job after 20 minutes
+NONCE=$(echo "$(date "+%Y%m%d%H%M") - ($(date +%M)%20)" | bc)
+
 # Echo only when not in parallel mode
 say() {
   if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
@@ -36,10 +40,11 @@ default-setup-command() {
 pip-install() {
   cd "$GITHUB_WORKSPACE"
 
-  # Don't use pip cache as it doesn't seem to help much.
+  # Pip cache saves at most about 20s on a good day
   # cache-restore pip
 
   say "::group::Install Python pacakges"
+  pip install --upgrade pip
   pip install -r requirements.txt
   pip install -r requirements-dev.txt
   pip install -e ".[postgres,mysql]"
@@ -52,15 +57,16 @@ pip-install() {
 npm-install() {
   cd "$GITHUB_WORKSPACE/superset-frontend"
 
-  cache-restore npm
+  # cache-restore npm
 
   say "::group::Install npm packages"
   echo "npm: $(npm --version)"
   echo "node: $(node --version)"
-  npm ci
+  rm -rf ./node_modules
+  npm install
   say "::endgroup::"
 
-  cache-save npm
+  # cache-save npm
 }
 
 build-assets() {
@@ -71,16 +77,6 @@ build-assets() {
   say "::endgroup::"
 }
 
-build-assets-cached() {
-  cache-restore assets
-  if [[ -f "$ASSETS_MANIFEST" ]]; then
-    echo 'Skip frontend build because static assets already exist.'
-  else
-    build-assets
-    cache-save assets
-  fi
-}
-
 build-instrumented-assets() {
   cd "$GITHUB_WORKSPACE/superset-frontend"
 
@@ -139,7 +135,7 @@ codecov() {
   local codecovScript="${HOME}/codecov.sh"
   # download bash script if needed
   if [[ ! -f "$codecovScript" ]]; then
-    curl -s https://codecov.io/bash > "$codecovScript"
+    curl -s https://codecov.io/bash >"$codecovScript"
   fi
   bash "$codecovScript" "$@"
   say "::endgroup::"
@@ -160,19 +156,22 @@ cypress-install() {
 # Run Cypress and upload coverage reports
 cypress-run() {
   cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
-  
+
   local page=$1
   local group=${2:-Default}
   local cypress="./node_modules/.bin/cypress run"
   local browser=${CYPRESS_BROWSER:-chrome}
 
+  export TERM="xterm"
+
   say "::group::Run Cypress for [$page]"
   if [[ -z $CYPRESS_RECORD_KEY ]]; then
     $cypress --spec "cypress/integration/$page" --browser "$browser"
   else
     # additional flags for Cypress dashboard recording
-    $cypress --spec "cypress/integration/$page" --browser "$browser" --record \
+    $cypress --spec "cypress/integration/$page" --browser "$browser" \
-      --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}"
+      --record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
+      --parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}"
   fi
 
   # don't add quotes to $record because we do want word splitting
@@ -186,7 +185,7 @@ cypress-run-all() {
   local flasklog="${HOME}/flask.log"
   local port=8081
 
-  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
   cypress-run "*/**/*"
@@ -205,7 +204,7 @@ cypress-run-all() {
 
   # Restart Flask with new configs
   kill $flaskProcessId
-  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
   cypress-run "sqllab/*" "Backend persist"

.github/workflows/caches.js

@@ -40,7 +40,7 @@ module.exports = {
   },
   npm: {
     path: [`${homeDirectory}/.npm`],
-    hashFiles: ['superset-frontend/package-lock.json'],
+    hashFiles: [`${workspaceDirectory}/superset-frontend/package-lock.json`],
   },
   assets: assetsConfig,
   // use separate cache for instrumented JS files and regular assets

.github/workflows/superset-e2e.yml

@@ -34,22 +34,20 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: '3.6'
 
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
-        # Run commands in parallel does help initial installation without cache
-        parallel: true
         run: |
           npm-install && build-instrumented-assets
           pip-install && setup-postgres && testdata
           cypress-install
 
     - name: Run Cypress
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       env:
         CYPRESS_BROWSER: ${{ matrix.browser }}
         CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

.github/workflows/superset-frontend.yml

@@ -10,7 +10,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: npm-install
     - name: lint

.github/workflows/superset-python.yml

@@ -14,11 +14,11 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
     - name: black
       run: black --check $(echo $PYTHON_LINT_TARGET)
     - name: mypy
@@ -38,11 +38,11 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
           pip-install
@@ -59,11 +59,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2.1.1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        uses: apache-superset/cached-dependencies@adc6f73
+        uses: apache-superset/cached-dependencies@b90713b
         with:
           run: |
             pip-install
@@ -100,11 +100,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
           pip-install
@@ -144,11 +144,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
           pip-install
@@ -181,11 +181,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2.1.1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
           pip-install

.github/workflows/superset-translations.yml

@@ -10,7 +10,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@adc6f73
+      uses: apache-superset/cached-dependencies@b90713b
       with:
         run: npm-install
     - name: lint

tests/schedules_test.py

@@ -62,7 +62,7 @@ class TestSchedules(SupersetTestCase):
             )
 
             # Pick up a random slice and dashboard
-            slce = db.session.query(Slice).all()[0]
+            slce = db.session.query(Slice).filter_by(slice_name="Participants").all()[0]
             dashboard = db.session.query(Dashboard).all()[0]
 
             dashboard_schedule = DashboardEmailSchedule(**cls.common_data)