feat(docs): auto-generate database documentation from lib.py (#36805)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/superset-docs-deploy.yml

@@ -1,6 +1,13 @@
 name: Docs Deployment
 
 on:
+  # Deploy after integration tests complete on master
+  workflow_run:
+    workflows: ["Python-Integration"]
+    types: [completed]
+    branches: [master]
+
+  # Also allow manual trigger and direct pushes to docs
   push:
     paths:
       - "docs/**"
@@ -30,9 +37,10 @@ jobs:
     name: Build & Deploy
     runs-on: ubuntu-24.04
     steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      - name: "Checkout ${{ github.event.workflow_run.head_sha || github.sha }}"
         uses: actions/checkout@v6
         with:
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
           persist-credentials: false
           submodules: recursive
       - name: Set up Node.js
@@ -58,6 +66,35 @@ jobs:
         working-directory: docs
         run: |
           yarn install --check-cache
+      - name: Download database diagnostics (if triggered by integration tests)
+        if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success'
+        uses: dawidd6/action-download-artifact@v6
+        continue-on-error: true
+        with:
+          workflow: superset-python-integrationtest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+          name: database-diagnostics
+          path: docs/src/data/
+      - name: Try to download latest diagnostics (for push/dispatch triggers)
+        if: github.event_name != 'workflow_run'
+        uses: dawidd6/action-download-artifact@v6
+        continue-on-error: true
+        with:
+          workflow: superset-python-integrationtest.yml
+          name: database-diagnostics
+          path: docs/src/data/
+          branch: master
+          search_artifacts: true
+          if_no_artifact_found: warn
+      - name: Use diagnostics artifact if available
+        working-directory: docs
+        run: |
+          if [ -f "src/data/databases-diagnostics.json" ]; then
+            echo "Using fresh diagnostics from integration tests"
+            mv src/data/databases-diagnostics.json src/data/databases.json
+          else
+            echo "Using committed databases.json (no artifact found)"
+          fi
       - name: yarn build
         working-directory: docs
         run: |
@@ -71,5 +108,5 @@ jobs:
           destination-github-username: "apache"
           destination-repository-name: "superset-site"
           target-branch: "asf-site"
-          commit-message: "deploying docs: ${{ github.event.head_commit.message }} (apache/superset@${{ github.sha }})"
+          commit-message: "deploying docs: ${{ github.event.head_commit.message || 'triggered by integration tests' }} (apache/superset@${{ github.event.workflow_run.head_sha || github.sha }})"
           user-email: dev@superset.apache.org

.github/workflows/superset-docs-verify.yml

@@ -4,17 +4,23 @@ on:
   pull_request:
     paths:
       - "docs/**"
+      - "superset/db_engine_specs/**"
       - ".github/workflows/superset-docs-verify.yml"
     types: [synchronize, opened, reopened, ready_for_review]
+  workflow_run:
+    workflows: ["Python-Integration"]
+    types: [completed]
 
 # cancel previous workflow jobs for PRs
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.head_sha || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   linkinator:
     # See docs here: https://github.com/marketplace/actions/linkinator
+    # Only run on pull_request, not workflow_run
+    if: github.event_name == 'pull_request'
     name: Link Checking
     runs-on: ubuntu-latest
     steps:
@@ -50,8 +56,11 @@ jobs:
             https://timbr.ai/,
             https://opensource.org/license/apache-2-0,
             https://www.plaidcloud.com/
-  build-deploy:
-    name: Build & Deploy
+
+  build-on-pr:
+    # Build docs when PR changes docs/** (uses committed databases.json)
+    if: github.event_name == 'pull_request'
+    name: Build (PR trigger)
     runs-on: ubuntu-24.04
     defaults:
       run:
@@ -75,3 +84,50 @@ jobs:
       - name: yarn build
         run: |
           yarn build
+
+  build-after-tests:
+    # Build docs after integration tests complete (uses fresh diagnostics)
+    # Only runs if integration tests succeeded
+    if: >
+      github.event_name == 'workflow_run' &&
+      github.event.workflow_run.conclusion == 'success'
+    name: Build (after integration tests)
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: docs
+    steps:
+      - name: "Checkout PR head: ${{ github.event.workflow_run.head_sha }}"
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+          persist-credentials: false
+          submodules: recursive
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: './docs/.nvmrc'
+      - name: yarn install
+        run: |
+          yarn install --check-cache
+      - name: Download database diagnostics from integration tests
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          workflow: superset-python-integrationtest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+          name: database-diagnostics
+          path: docs/src/data/
+      - name: Use fresh diagnostics
+        run: |
+          if [ -f "src/data/databases-diagnostics.json" ]; then
+            echo "Using fresh diagnostics from integration tests"
+            mv src/data/databases-diagnostics.json src/data/databases.json
+          else
+            echo "Warning: No diagnostics artifact found, using committed data"
+          fi
+      - name: yarn typecheck
+        run: |
+          yarn typecheck
+      - name: yarn build
+        run: |
+          yarn build

.github/workflows/superset-python-integrationtest.yml

@@ -73,6 +73,36 @@ jobs:
           flags: python,mysql
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+      - name: Generate database diagnostics for docs
+        if: steps.check.outputs.python
+        env:
+          SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+          SUPERSET__SQLALCHEMY_DATABASE_URI: |
+            mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
+        run: |
+          python -c "
+          import json
+          from superset.app import create_app
+          from superset.db_engine_specs.lib import generate_yaml_docs
+          app = create_app()
+          with app.app_context():
+              docs = generate_yaml_docs()
+              # Wrap in the expected format
+              output = {
+                  'generated': '$(date -Iseconds)',
+                  'databases': docs
+              }
+              with open('databases-diagnostics.json', 'w') as f:
+                  json.dump(output, f, indent=2, default=str)
+              print(f'Generated diagnostics for {len(docs)} databases')
+          "
+      - name: Upload database diagnostics artifact
+        if: steps.check.outputs.python
+        uses: actions/upload-artifact@v4
+        with:
+          name: database-diagnostics
+          path: databases-diagnostics.json
+          retention-days: 7
   test-postgres:
     runs-on: ubuntu-24.04
     strategy: