[Build] Collect frontend code coverage from Cypress tests (#9555)

* build: collect code coverage from Cypress

Collect frontend code coverage reports from Cypress tests and add
proper tagging for all tests.

* Fix bash script lint error from shellcheck

* Revert Cypress to 4.3.0 to see if it fixes a failing test

.github/workflows/bashlib.sh

@@ -15,10 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+set -e
+
+GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
+ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
 
 # Echo only when not in parallel mode
 say() {
-  if [[ ${INPUT_PARALLEL^^} != 'TRUE' ]]; then
+  if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
     echo "$1"
   fi
 }
@@ -67,41 +71,27 @@ build-assets() {
   say "::endgroup::"
 }
 
-npm-build() {
-  if [[ $1 = '--no-cache' ]]; then
-    build-assets
+build-assets-cached() {
+  cache-restore assets
+  if [[ -f "$ASSETS_MANIFEST" ]]; then
+    echo 'Skip frontend build because static assets already exist.'
   else
-    cache-restore assets
-    if [[ -f $GITHUB_WORKSPACE/superset/static/assets/manifest.json ]]; then
-      echo 'Skip frontend build because static assets already exist.'
-    else
-      build-assets
-      cache-save assets
-    fi
+    build-assets
+    cache-save assets
   fi
 }
 
-cypress-install() {
-  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+build-instrumented-assets() {
+  cd "$GITHUB_WORKSPACE/superset-frontend"
 
-  cache-restore cypress
-
-  say "::group::Install Cypress"
-  npm ci
-  say "::endgroup::"
-
-  cache-save cypress
-}
-
-testdata() {
-  cd "$GITHUB_WORKSPACE"
-  say "::group::Load test data"
-  # must specify PYTHONPATH to make `tests.superset_test_config` importable
-  export PYTHONPATH="$GITHUB_WORKSPACE"
-  superset db upgrade
-  superset load_test_users
-  superset load_examples --load-test-data
-  superset init
+  say "::group::Build static assets with JS instrumented for test coverage"
+  cache-restore instrumented-assets
+  if [[ -f "$ASSETS_MANIFEST" ]]; then
+    echo 'Skip frontend build because instrumented static assets already exist.'
+  else
+    npm run build-instrumented -- --no-progress
+    cache-save instrumented-assets
+  fi
   say "::endgroup::"
 }
 
@@ -131,3 +121,100 @@ setup-mysql() {
 EOF
   say "::endgroup::"
 }
+
+testdata() {
+  cd "$GITHUB_WORKSPACE"
+  say "::group::Load test data"
+  # must specify PYTHONPATH to make `tests.superset_test_config` importable
+  export PYTHONPATH="$GITHUB_WORKSPACE"
+  superset db upgrade
+  superset load_test_users
+  superset load_examples --load-test-data
+  superset init
+  say "::endgroup::"
+}
+
+codecov() {
+  say "::group::Upload code coverage"
+  local codecovScript="${HOME}/codecov.sh"
+  # download bash script if needed
+  if [[ ! -f "$codecovScript" ]]; then
+    curl -s https://codecov.io/bash > "$codecovScript"
+  fi
+  bash "$codecovScript" "$@"
+  say "::endgroup::"
+}
+
+cypress-install() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+
+  cache-restore cypress
+
+  say "::group::Install Cypress"
+  npm ci
+  say "::endgroup::"
+
+  cache-save cypress
+}
+
+# Run Cypress and upload coverage reports
+cypress-run() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+  
+  local page=$1
+  local group=${2:-Default}
+  local cypress="./node_modules/.bin/cypress run"
+  local browser=${CYPRESS_BROWSER:-chrome}
+
+  say "::group::Run Cypress for [$page]"
+  if [[ -z $CYPRESS_RECORD_KEY ]]; then
+    $cypress --spec "cypress/integration/$page" --browser "$browser"
+  else
+    # additional flags for Cypress dashboard recording
+    $cypress --spec "cypress/integration/$page" --browser "$browser" --record \
+      --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}"
+  fi
+
+  # don't add quotes to $record because we do want word splitting
+  say "::endgroup::"
+}
+
+cypress-run-all() {
+  # Start Flask and run it in background
+  # --no-debugger means disable the interactive debugger on the 500 page
+  # so errors can print to stderr.
+  local flasklog="${HOME}/flask.log"
+  local port=8081
+
+  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "*/*"
+
+  # Upload code coverage separately so each page can have separate flags
+  # -c will clean existing coverage reports, -F means add flags
+  codecov -cF "cypress"
+
+  # After job is done, print out Flask log for debugging
+  say "::group::Flask log for default run"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # Rerun SQL Lab tests with backend persist enabled
+  export SUPERSET_CONFIG=tests.superset_test_config_sqllab_backend_persist
+
+  # Restart Flask with new configs
+  kill $flaskProcessId
+  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "sqllab/*" "Backend persist"
+  codecov -cF "cypress"
+
+  say "::group::Flask log for backend persist"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # make sure the program exits
+  kill $flaskProcessId
+}

.github/workflows/caches.js

@@ -21,6 +21,17 @@
 const workspaceDirectory = process.env.GITHUB_WORKSPACE;
 const homeDirectory = process.env.HOME;
 
+const assetsConfig = {
+  path: [`${workspaceDirectory}/superset/static/assets`],
+  hashFiles: [
+    `${workspaceDirectory}/superset-frontend/src/**/*`,
+    `${workspaceDirectory}/superset-frontend/*.js`,
+    `${workspaceDirectory}/superset-frontend/*.json`,
+  ],
+  // dont use restore keys as it may give an invalid older build
+  restoreKeys: '',
+};
+
 // Multi-layer cache definition
 module.exports = {
   pip: {
@@ -31,18 +42,11 @@ module.exports = {
     path: [`${homeDirectory}/.npm`],
     hashFiles: ['superset-frontend/package-lock.json'],
   },
-  assets: {
-    path: [
-      `${workspaceDirectory}/superset/static/assets`,
-    ],
-    hashFiles: [
-      `${workspaceDirectory}/superset-frontend/src/**/*`,
-      `${workspaceDirectory}/superset-frontend/*.json`,
-      `${workspaceDirectory}/superset-frontend/*.js`,
-    ],
-    // dont use restore keys as it may give an invalid older build
-    restoreKeys: ''
-  },
+  assets: assetsConfig,
+  // use separate cache for instrumented JS files and regular assets
+  // one is built with `npm run build`,
+  // another is built with `npm run build-instrumented`
+  'instrumented-assets': assetsConfig,
   cypress: {
     path: [`${homeDirectory}/.cache/Cypress`],
     hashFiles: [

.github/workflows/superset-e2e.yml

@@ -7,7 +7,7 @@ jobs:
     name: Cypress
     runs-on: ubuntu-18.04
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         browser: ['chrome']
     env:
@@ -17,7 +17,6 @@ jobs:
         postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
       PYTHONPATH: ${{ github.workspace }}
       REDIS_PORT: 16379
-      CI: github-actions
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     services:
       postgres:
@@ -40,42 +39,19 @@ jobs:
         python-version: '3.6'
 
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         # Run commands in parallel does help initial installation without cache
         parallel: true
         run: |
-          npm-install && npm-build
+          npm-install && build-instrumented-assets
           pip-install && setup-postgres && testdata
           cypress-install
 
-    - name: Cypress run all
+    - name: Run Cypress
+      uses: apache-superset/cached-dependencies@adc6f73
       env:
-        CYPRESS_GROUP: Default
-        CYPRESS_PATH: 'cypress/integration/*/*'
-      run: |
-        # Start Flask and run Cypress
-
-        # --no-debugger means disable the interactive debugger on the 500 page
-        # so errors can print to stderr.
-        flask run --no-debugger --with-threads -p 8081 &
-
-        sleep 3 # wait for the Flask app to start
-
-        cd ${{ github.workspace }}/superset-frontend/cypress-base/
-        npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false
-
-    - name: Cypress run SQL Lab (with backend persist)
-      env:
-        SUPERSET_CONFIG: tests.superset_test_config_sqllab_backend_persist
-        CYPRESS_GROUP: Backend persist
-        CYPRESS_PATH: 'cypress/integration/sqllab/*'
-      run: |
-        # Start Flask with alternative config and run Cypress
-
-        killall python # exit the running Flask app
-        flask run --no-debugger --with-threads -p 8081 &
-        sleep 3 # wait for the Flask app to start
-
-        cd ${{ github.workspace }}/superset-frontend/cypress-base/
-        npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false
+        CYPRESS_BROWSER: ${{ matrix.browser }}
+        CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+      with:
+        run: cypress-run-all

.github/workflows/superset-frontend.yml

@@ -6,13 +6,11 @@ jobs:
   frontend-build:
     name: build
     runs-on: ubuntu-18.04
-    env:
-      CI: github-actions
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: npm-install
     - name: eslint
@@ -26,4 +24,4 @@ jobs:
     - name: Upload code coverage
       working-directory: ./superset-frontend
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,javascript
+        bash <(curl -s https://codecov.io/bash) -cF javascript

.github/workflows/superset-python.yml

@@ -10,7 +10,6 @@ jobs:
         python-version: [3.6]
     env:
       PYTHON_LINT_TARGET: setup.py superset tests
-      CI: github-actions
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
@@ -19,7 +18,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
     - name: black
       run: black --check $(echo $PYTHON_LINT_TARGET)
     - name: mypy
@@ -62,7 +61,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -75,7 +74,7 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,postgres
+        bash <(curl -s https://codecov.io/bash) -cF python
 
   test-mysql:
     runs-on: ubuntu-18.04
@@ -105,7 +104,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -118,7 +117,7 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,mysql
+        bash <(curl -s https://codecov.io/bash) -cF python
 
   test-sqlite:
     runs-on: ubuntu-18.04
@@ -141,7 +140,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -154,4 +153,4 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,sqlite
+        bash <(curl -s https://codecov.io/bash) -cF python