diff --git a/.asf.yaml b/.asf.yaml index 39292a6e85c..90f75523f67 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -83,6 +83,7 @@ github: - cypress-matrix (5, chrome) - dependency-review - frontend-build + - playwright-tests (chromium) - pre-commit (current) - pre-commit (previous) - test-mysql diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh index df4fb99c103..1289d07259e 100644 --- a/.github/workflows/bashlib.sh +++ b/.github/workflows/bashlib.sh @@ -195,6 +195,7 @@ playwright-install() { playwright-run() { local APP_ROOT=$1 + local TEST_PATH=$2 # Start Flask from the project root (same as Cypress) cd "$GITHUB_WORKSPACE" @@ -238,8 +239,26 @@ playwright-run() { say "::group::Run Playwright tests" echo "Running Playwright with baseURL: ${PLAYWRIGHT_BASE_URL}" - npx playwright test auth/login --reporter=github --output=playwright-results - local status=$? + if [ -n "$TEST_PATH" ]; then + # Check if there are any test files in the specified path + if ! find "playwright/tests/${TEST_PATH}" -name "*.spec.ts" -type f 2>/dev/null | grep -q .; then + echo "No test files found in ${TEST_PATH} - skipping test run" + say "::endgroup::" + kill $flaskProcessId + return 0 + fi + echo "Running tests: ${TEST_PATH}" + # Set INCLUDE_EXPERIMENTAL=true to allow experimental tests to run + export INCLUDE_EXPERIMENTAL=true + npx playwright test "${TEST_PATH}" --output=playwright-results + local status=$? + # Unset to prevent leaking into subsequent commands + unset INCLUDE_EXPERIMENTAL + else + echo "Running all required tests (experimental/ excluded via playwright.config.ts)" + npx playwright test --output=playwright-results + local status=$? + fi say "::endgroup::" # After job is done, print out Flask log for debugging diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 70d6b28364a..1e074980498 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -151,3 +151,118 @@ jobs: with: path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots name: cypress-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}-${{ matrix.parallel_id }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} + + playwright-tests: + runs-on: ubuntu-22.04 + permissions: + contents: read + pull-requests: read + strategy: + fail-fast: false + matrix: + browser: ["chromium"] + app_root: ["", "/app/prefix"] + env: + SUPERSET_ENV: development + SUPERSET_CONFIG: tests.integration_tests.superset_test_config + SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset + PYTHONPATH: ${{ github.workspace }} + REDIS_PORT: 16379 + GITHUB_TOKEN: ${{ github.token }} + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + - 15432:5432 + redis: + image: redis:7-alpine + ports: + - 16379:6379 + steps: + # ------------------------------------------------------- + # Conditional checkout based on context (same as Cypress workflow) + - name: Checkout for push or pull_request event + if: github.event_name == 'push' || github.event_name == 'pull_request' + uses: actions/checkout@v5 + with: + persist-credentials: false + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + - name: Checkout using ref (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != '' + uses: actions/checkout@v5 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref }} + submodules: recursive + - name: Checkout using PR ID (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != '' + uses: actions/checkout@v5 + with: + persist-credentials: false + ref: refs/pull/${{ github.event.inputs.pr_id }}/merge + submodules: recursive + # ------------------------------------------------------- + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Python + uses: ./.github/actions/setup-backend/ + if: steps.check.outputs.python || steps.check.outputs.frontend + - name: Setup postgres + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + with: + run: setup-postgres + - name: Import test data + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + with: + run: testdata + - name: Setup Node.js + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: actions/setup-node@v5 + with: + node-version-file: './superset-frontend/.nvmrc' + - name: Install npm dependencies + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + with: + run: npm-install + - name: Build javascript packages + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + with: + run: build-instrumented-assets + - name: Install Playwright + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + with: + run: playwright-install + - name: Run Playwright (Required Tests) + if: steps.check.outputs.python || steps.check.outputs.frontend + uses: ./.github/actions/cached-dependencies + env: + NODE_OPTIONS: "--max-old-space-size=4096" + with: + run: playwright-run "${{ matrix.app_root }}" + - name: Set safe app root + if: failure() + id: set-safe-app-root + run: | + APP_ROOT="${{ matrix.app_root }}" + SAFE_APP_ROOT=${APP_ROOT//\//_} + echo "safe_app_root=$SAFE_APP_ROOT" >> $GITHUB_OUTPUT + - name: Upload Playwright Artifacts + uses: actions/upload-artifact@v4 + if: failure() + with: + path: | + ${{ github.workspace }}/superset-frontend/playwright-results/ + ${{ github.workspace }}/superset-frontend/test-results/ + name: playwright-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} diff --git a/.github/workflows/superset-playwright.yml b/.github/workflows/superset-playwright.yml index 0a94ef89d0c..f9ca2c09aa9 100644 --- a/.github/workflows/superset-playwright.yml +++ b/.github/workflows/superset-playwright.yml @@ -1,4 +1,4 @@ -name: Playwright E2E Tests +name: Playwright Experimental Tests on: push: @@ -23,9 +23,10 @@ concurrency: cancel-in-progress: true jobs: - playwright-tests: + # NOTE: Required Playwright tests are in superset-e2e.yml (E2E / playwright-tests) + # This workflow contains only experimental tests that run in shadow mode + playwright-tests-experimental: runs-on: ubuntu-22.04 - # Allow workflow to succeed even if tests fail during shadow mode continue-on-error: true permissions: contents: read @@ -117,13 +118,13 @@ jobs: uses: ./.github/actions/cached-dependencies with: run: playwright-install - - name: Run Playwright + - name: Run Playwright (Experimental Tests) if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: NODE_OPTIONS: "--max-old-space-size=4096" with: - run: playwright-run ${{ matrix.app_root }} + run: playwright-run "${{ matrix.app_root }}" experimental/ - name: Set safe app root if: failure() id: set-safe-app-root @@ -138,4 +139,4 @@ jobs: path: | ${{ github.workspace }}/superset-frontend/playwright-results/ ${{ github.workspace }}/superset-frontend/test-results/ - name: playwright-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} + name: playwright-experimental-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }} diff --git a/superset-frontend/cypress-base/cypress/e2e/auth/login.test.ts b/superset-frontend/cypress-base/cypress/e2e/auth/login.test.ts deleted file mode 100644 index 064167d82d9..00000000000 --- a/superset-frontend/cypress-base/cypress/e2e/auth/login.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import { LOGIN } from 'cypress/utils/urls'; - -function interceptLogin() { - cy.intercept('POST', '**/login/').as('login'); -} - -describe('Login view', () => { - beforeEach(() => { - cy.visit(LOGIN); - }); - - it('should redirect to login with incorrect username and password', () => { - interceptLogin(); - cy.getBySel('login-form').should('be.visible'); - cy.getBySel('username-input').type('admin'); - cy.getBySel('password-input').type('wrongpassword'); - cy.getBySel('login-button').click(); - cy.wait('@login'); - cy.url().should('include', LOGIN); - }); - - it('should login with correct username and password', () => { - interceptLogin(); - cy.getBySel('login-form').should('be.visible'); - cy.getBySel('username-input').type('admin'); - cy.getBySel('password-input').type('general'); - cy.getBySel('login-button').click(); - cy.wait('@login'); - cy.getCookies().should('have.length', 1); - }); -}); diff --git a/superset-frontend/playwright.config.ts b/superset-frontend/playwright.config.ts index f8170459c23..585ccbb96f7 100644 --- a/superset-frontend/playwright.config.ts +++ b/superset-frontend/playwright.config.ts @@ -26,6 +26,13 @@ export default defineConfig({ // Test directory testDir: './playwright/tests', + // Conditionally ignore experimental tests based on env var + // When INCLUDE_EXPERIMENTAL=true, experimental tests are included + // Otherwise, they are excluded (default for required tests) + testIgnore: process.env.INCLUDE_EXPERIMENTAL + ? undefined + : '**/experimental/**', + // Timeout settings timeout: 30000, expect: { timeout: 8000 }, diff --git a/superset-frontend/playwright/tests/experimental/README.md b/superset-frontend/playwright/tests/experimental/README.md new file mode 100644 index 00000000000..9647fb23960 --- /dev/null +++ b/superset-frontend/playwright/tests/experimental/README.md @@ -0,0 +1,70 @@ + + +# Experimental Playwright Tests + +This directory contains Playwright tests that are still under development or validation. + +## Purpose + +Tests in this directory run in "shadow mode" with `continue-on-error: true` in CI: +- Failures do NOT block PR merges +- Allows tests to run in CI to validate stability before promotion +- Provides visibility into test reliability over time + +## Promoting Tests to Stable + +Once a test has proven stable (no false positives/negatives over sufficient time): + +1. Move the test file out of `experimental/` to the appropriate feature directory: + ```bash + # From the repository root: + git mv superset-frontend/playwright/tests/experimental/dashboard/test.spec.ts \ + superset-frontend/playwright/tests/dashboard/ + + # Or from the superset-frontend/ directory: + git mv playwright/tests/experimental/dashboard/test.spec.ts \ + playwright/tests/dashboard/ + ``` + +2. The test will automatically become required for merge + +## Test Organization + +Organize tests by feature area: +- `auth/` - Authentication and authorization tests +- `dashboard/` - Dashboard functionality tests +- `explore/` - Chart builder tests +- `sqllab/` - SQL Lab tests +- etc. + +## Running Tests + +```bash +# Run all experimental tests (requires INCLUDE_EXPERIMENTAL env var) +INCLUDE_EXPERIMENTAL=true npm run playwright:test -- experimental/ + +# Run specific experimental test +INCLUDE_EXPERIMENTAL=true npm run playwright:test -- experimental/dashboard/test.spec.ts + +# Run in UI mode for debugging +INCLUDE_EXPERIMENTAL=true npm run playwright:ui -- experimental/ +``` + +**Note**: The `INCLUDE_EXPERIMENTAL=true` environment variable is required because experimental tests are filtered out by default in `playwright.config.ts`. Without it, Playwright will report "No tests found".