chore: 2nd try - simplify python dependencies (#27505)

.github/workflows/pre-commit.yml

@@ -26,24 +26,9 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: |
-            requirements/base.txt
-            requirements/integration.txt
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
-            pip install -r requirements/integration.txt
-      # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
+        uses: ./.github/actions/setup-backend/
       - name: Enable brew and helm-docs
+        # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
         run: |
           echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
           eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

.github/workflows/superset-applitool-cypress.yml

@@ -58,23 +58,7 @@ jobs:
           submodules: recursive
           ref: master
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-      - name: OS dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: apt-get-install
-      - name: Install python dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
-      - name: Setup postgres
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: setup-postgres
+        uses: ./.github/actions/setup-backend/
       - name: Import test data
         uses: ./.github/actions/cached-dependencies
         with:
@@ -91,6 +75,11 @@ jobs:
         uses: ./.github/actions/cached-dependencies
         with:
           run: build-instrumented-assets
+      - name: Setup Postgres
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
       - name: Install cypress
         uses: ./.github/actions/cached-dependencies
         with:

.github/workflows/superset-cli.yml

@@ -52,22 +52,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: superset init
         if: steps.check.outcome == 'failure'
         run: |

.github/workflows/superset-e2e.yml

@@ -69,21 +69,15 @@ jobs:
         run: ./scripts/ci_check_no_file_changes.sh python frontend
       - name: Setup Python
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
+        uses: ./.github/actions/setup-backend/
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: apt-get-install
-      - name: Install python dependencies
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
       - name: Setup postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-python-integrationtest.yml

@@ -51,21 +51,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup MySQL
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -78,7 +72,6 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           bash .github/workflows/codecov.sh -c -F python -F mysql
-
   test-postgres:
     runs-on: ubuntu-20.04
     strategy:
@@ -117,21 +110,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -177,21 +164,16 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
+            # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
         if: steps.check.outcome == 'failure'

.github/workflows/superset-python-misc.yml

@@ -38,21 +38,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
       - name: pylint
         if: steps.check.outcome == 'failure'
         # `-j 0` run Pylint in parallel
@@ -70,18 +59,8 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/base.txt"
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

.github/workflows/superset-python-presto-hive.yml

@@ -67,22 +67,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -144,22 +135,13 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &

.github/workflows/superset-python-unittest.yml

@@ -46,27 +46,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      # TODO: separated requirements.txt file just for unit tests
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            mkdir ${{ github.workspace }}/.temp
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         env:
           SUPERSET_TESTENV: true
+          SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage

.github/workflows/superset-translations.yml

@@ -47,7 +47,7 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies