From 5948dac88940dbe38275b002d30ad33b1b55e698 Mon Sep 17 00:00:00 2001
From: John Bodley <4567245+john-bodley@users.noreply.github.com>
Date: Wed, 26 Jun 2024 12:10:05 -0700
Subject: [PATCH] chore(ci): Start Celery worker as a background process
 (#29371)

---
 .github/workflows/bashlib.sh                   | 14 ++++++++++++++
 .../superset-python-integrationtest.yml        | 18 ++++++++++++------
 .../workflows/superset-python-presto-hive.yml  | 12 ++++++++----
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh
index 82840becb83..31e7d04b23b 100644
--- a/.github/workflows/bashlib.sh
+++ b/.github/workflows/bashlib.sh
@@ -117,6 +117,20 @@ testdata() {
   say "::endgroup::"
 }
 
+celery-worker() {
+  cd "$GITHUB_WORKSPACE"
+  say "::group::Start Celery worker"
+  # must specify PYTHONPATH to make `tests.superset_test_config` importable
+  export PYTHONPATH="$GITHUB_WORKSPACE"
+  celery \
+    --app=superset.tasks.celery_app:app \
+    worker \
+      --concurrency=2 \
+      --detach \
+      --optimization=fair
+  say "::endgroup::"
+}
+
 cypress-install() {
   cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
 
diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
index 3f43bef88c1..80b2a3b98b7 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -58,9 +58,11 @@ jobs:
         uses: ./.github/actions/cached-dependencies
         with:
           run: setup-mysql
-      - name: Run celery
+      - name: Start Celery worker
         if: steps.check.outputs.python
-        run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: celery-worker
       - name: Python integration tests (MySQL)
         if: steps.check.outputs.python
         run: |
@@ -117,9 +119,11 @@ jobs:
         with:
           run: |
             setup-postgres
-      - name: Run celery
+      - name: Start Celery worker
         if: steps.check.outputs.python
-        run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: celery-worker
       - name: Python integration tests (PostgreSQL)
         if: steps.check.outputs.python
         run: |
@@ -167,9 +171,11 @@ jobs:
           run: |
             # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
-      - name: Run celery
+      - name: Start Celery worker
         if: steps.check.outputs.python
-        run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: celery-worker
       - name: Python integration tests (SQLite)
         if: steps.check.outputs.python
         run: |
diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml
index 30dbaf7d40c..6ab65430b40 100644
--- a/.github/workflows/superset-python-presto-hive.yml
+++ b/.github/workflows/superset-python-presto-hive.yml
@@ -67,9 +67,11 @@ jobs:
           run: |
             echo "${{ steps.check.outputs.python }}"
             setup-postgres
-      - name: Run celery
+      - name: Start Celery worker
         if: steps.check.outputs.python
-        run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: celery-worker
       - name: Python unit tests (PostgreSQL)
         if: steps.check.outputs.python
         run: |
@@ -132,9 +134,11 @@ jobs:
         uses: ./.github/actions/cached-dependencies
         with:
           run: setup-postgres
-      - name: Run celery
+      - name: Start Celery worker
         if: steps.check.outputs.python
-        run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: celery-worker
       - name: Python unit tests (PostgreSQL)
         if: steps.check.outputs.python
         run: |