chore: Adding pip-compile-multi et al. (#10499)

* chore: Adding pip-compile-multi et al

* Specify requirements.txt path for fossa

* [ci] Fixing CI

Co-authored-by: John Bodley <john.bodley@airbnb.com>
Co-authored-by: Jesse Yang <jesse.yang@airbnb.com>

.github/workflows/bashlib.sh

@@ -33,24 +33,21 @@ say() {
 
 # default command to run when the `run` input is empty
 default-setup-command() {
-  pip-install
+  apt-get-install
+  pip-upgrade
 }
 
-# install python dependencies
-pip-install() {
-  cd "$GITHUB_WORKSPACE"
+apt-get-install() {
+    say "::group::apt-get install dependencies"
+    sudo apt-get update && sudo apt-get install --yes \
+      libsasl2-dev
+    say "::endgroup::"
+}
 
-  # Pip cache saves at most about 20s on a good day
-  # cache-restore pip
-
-  say "::group::Install Python pacakges"
+pip-upgrade() {
+  say "::group::Upgrade pip"
   pip install --upgrade pip
-  pip install -r requirements.txt
-  pip install -r requirements-dev.txt
-  pip install -e ".[postgres,mysql]"
   say "::endgroup::"
-
-  # cache-save pip
 }
 
 # prepare (lint and build) frontend code
@@ -123,6 +120,7 @@ testdata() {
   say "::group::Load test data"
   # must specify PYTHONPATH to make `tests.superset_test_config` importable
   export PYTHONPATH="$GITHUB_WORKSPACE"
+  pip install -e .
   superset db upgrade
   superset load_test_users
   superset load_examples --load-test-data

.github/workflows/caches.js

@@ -36,7 +36,7 @@ const assetsConfig = {
 module.exports = {
   pip: {
     path: [`${homeDirectory}/.cache/pip`],
-    hashFiles: [`${workspaceDirectory}/requirements*.txt`],
+    hashFiles: [`${workspaceDirectory}/requirements/*.txt`],
   },
   npm: {
     path: [`${homeDirectory}/.npm`],

.github/workflows/license-check.yml

@@ -33,7 +33,7 @@ jobs:
 
         EOF
 
-        if [[ "${FILES}" =~ (.*package*\.json|requirements.*\.txt|setup\.py) ]]; then
+        if [[ "${FILES}" =~ (.*package*\.json|requirements/*\.txt|setup\.py) ]]; then
           echo "Detected dependency changes... running fossa check"
 
           ./scripts/fossa.sh

.github/workflows/superset-e2e.yml

@@ -37,15 +37,18 @@ jobs:
       uses: actions/setup-python@v2.1.1
       with:
         python-version: '3.6'
-
     - name: Install dependencies
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          npm-install && build-instrumented-assets
-          pip-install && setup-postgres && testdata
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
+          setup-postgres
+          testdata
+          npm-install
+          build-instrumented-assets
           cypress-install
-
     - name: Run Cypress
       uses: apache-superset/cached-dependencies@b90713b
       env:

.github/workflows/superset-python.yml

@@ -8,8 +8,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6]
-    env:
-      PYTHON_LINT_TARGET: setup.py superset tests
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
@@ -19,16 +17,37 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       uses: apache-superset/cached-dependencies@b90713b
-    - name: black
-      run: black --check $(echo $PYTHON_LINT_TARGET)
-    - name: mypy
-      run: mypy $(echo $PYTHON_LINT_TARGET)
-    - name: isort
-      run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET)
+      with:
+        run: |
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
     - name: pylint
       # `-j 0` run Pylint in parallel
       run: pylint -j 0 superset
 
+  pre-commit:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.6]
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2.1.1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@b90713b
+      with:
+        run: |
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/integration.txt
+    - name: pre-commit
+      run: pre-commit run --all-files
+
   docs:
     runs-on: ubuntu-18.04
     strategy:
@@ -45,8 +64,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
-          pip install -r docs/requirements.txt
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/documentation.txt
     - name: Build documentation
       run: sphinx-build -b html docs _build/html -W
 
@@ -66,8 +86,9 @@ jobs:
         uses: apache-superset/cached-dependencies@b90713b
         with:
           run: |
-            pip-install
-            pip install -r docs/requirements.txt
+            apt-get-install
+            pip-upgrade
+            pip install -r requirements/documentation.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
 
@@ -107,7 +128,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           setup-postgres
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@@ -151,7 +174,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           setup-mysql
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@@ -188,7 +213,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           mkdir ${{ github.workspace }}/.temp
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &