fix: docker refactor (#31385)

.dockerignore

@@ -45,3 +45,4 @@ superset-frontend/coverage/
 superset/static/assets/
 superset-websocket/dist/
 venv
+.venv

.github/workflows/ephemeral-env.yml

@@ -148,9 +148,20 @@ jobs:
       - name: Setup supersetbot
         uses: ./.github/actions/setup-supersetbot/
 
+      - name: Try to login to DockerHub
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
+        continue-on-error: true
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Build ephemeral env image
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           supersetbot docker \
             --preset ci \

Dockerfile

@@ -22,26 +22,29 @@ ARG PY_VER=3.10-slim-bookworm
 
 # If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
 ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
-FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
 
+######################################################################
+# superset-node used for building frontend assets
+######################################################################
+FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false"           # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
+
+COPY docker/ /app/docker/
 # Arguments for build configuration
 ARG NPM_BUILD_CMD="build"
-ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
-ARG DEV_MODE="false"           # Skip frontend build in dev mode
-ARG INCLUDE_CHROMIUM="true"    # Include headless Chromium for alerts & reports
-ARG INCLUDE_FIREFOX="false"    # Include headless Firefox if enabled
 
 # Install system dependencies required for node-gyp
-RUN --mount=type=bind,source=./docker,target=/docker \
+RUN /app/docker/apt-install.sh build-essential python3 zstd
-    /docker/apt-install.sh build-essential python3 zstd
 
 # Define environment variables for frontend build
 ENV BUILD_CMD=${NPM_BUILD_CMD} \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 
 # Run the frontend memory monitoring script
-RUN --mount=type=bind,source=./docker,target=/docker \
+RUN /app/docker/frontend-mem-nag.sh
-    /docker/frontend-mem-nag.sh
 
 WORKDIR /app/superset-frontend
 
@@ -49,6 +52,9 @@ WORKDIR /app/superset-frontend
 RUN mkdir -p /app/superset/static/assets \
              /app/superset/translations
 
+# Copy translation files
+COPY superset/translations /app/superset/translations
+
 # Mount package files and install dependencies if not in dev mode
 RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
     --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
@@ -61,41 +67,28 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
 # Runs the webpack build process
 COPY superset-frontend /app/superset-frontend
 
-
-# Copy translation files
-COPY superset/translations /app/superset/translations
-
 # Build the frontend if not in dev mode
 RUN if [ "$DEV_MODE" = "false" ]; then \
-        BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
+        echo "Running 'npm run ${BUILD_CMD}'"; \
+        if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
+            npm run build-translation; \
+        fi; \
+        npm run ${BUILD_CMD}; \
     else \
         echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
-    fi
+    fi && \
-
+    rm -rf /app/superset/translations/*/*/*.po
-# Compile .json files from .po translations (if required) and clean up .po files
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
-        npm run build-translation; \
-    else \
-        echo "Skipping translations as requested by build flag"; \
-    fi \
-	# removing translations files regardless
-    && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
-              /app/superset/translations/messages.pot
 
 
-# Transition to Python base image
+######################################################################
+# Base python layer
+######################################################################
 FROM python:${PY_VER} AS python-base
-RUN pip install --no-cache-dir --upgrade setuptools pip uv
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false"           # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
 
-######################################################################
-# Final lean image...
-######################################################################
-FROM python-base AS lean
-
-# Build argument for including translations
-ARG BUILD_TRANSLATIONS="false"
-
-WORKDIR /app
 ENV LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
     SUPERSET_ENV=production \
@@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \
     SUPERSET_HOME="/app/superset_home" \
     SUPERSET_PORT=8088
 
+
+RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset
+
+# Some bash scripts needed throughout the layers
+COPY --chmod=755 docker/*.sh /app/docker/
+
+RUN pip install --no-cache-dir --upgrade setuptools pip uv
+
+# Using uv as it's faster/simpler than pip
+RUN uv venv /app/.venv
+ENV PATH="/app/.venv/bin:${PATH}"
+
+# Install Playwright and optionally setup headless browsers
+ARG INCLUDE_CHROMIUM="true"
+ARG INCLUDE_FIREFOX="false"
+RUN --mount=type=cache,target=/root/.cache/pip \
+    if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
+        pip install playwright && \
+        playwright install-deps && \
+        if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \
+        if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \
+    else \
+        echo "Skipping browser installation"; \
+    fi
+
+######################################################################
+# Python translation compiler layer
+######################################################################
+FROM python-base AS python-translation-compiler
+
+# Install Python dependencies using docker/pip-install.sh
+COPY requirements/translations.txt requirements/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /app/docker/pip-install.sh -r requirements/translations.txt
+
+COPY superset/translations/ /app/translations_mo/
+RUN pybabel compile -d /app/translations_mo | true && \
+    rm -f /app/translations_mo/*/*/*.po
+
+######################################################################
+# Python APP common layer
+######################################################################
+FROM python-base AS python-common
+# Copy the entrypoints, make them executable in userspace
+COPY --chmod=755 docker/entrypoints /app/docker/entrypoints
+
+WORKDIR /app
 # Set up necessary directories and user
-RUN --mount=type=bind,source=./docker,target=/docker \
+RUN mkdir -p \
-    mkdir -p ${PYTHONPATH} \
+      ${SUPERSET_HOME} \
+      ${PYTHONPATH} \
       superset/static \
       requirements \
       superset-frontend \
       apache_superset.egg-info \
       requirements \
-    && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
+    && touch superset/static/version_info.json
-    && /docker/apt-install.sh \
-        curl \
-        libsasl2-dev \
-        libsasl2-modules-gssapi-mit \
-        libpq-dev \
-        libecpg-dev \
-        libldap2-dev \
-    && touch superset/static/version_info.json \
-    && chown -R superset:superset ./* \
-    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Copy required files for Python build
-COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
+COPY pyproject.toml setup.py MANIFEST.in README.md ./
-COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
+COPY superset-frontend/package.json superset-frontend/
-COPY --chown=superset:superset requirements/base.txt requirements/
+COPY scripts/check-env.py scripts/
-COPY --chown=superset:superset scripts/check-env.py scripts/
+
+# keeping for backward compatibility
+COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/
+
+# Some debian libs
+RUN /app/docker/apt-install.sh \
+      curl \
+      libsasl2-dev \
+      libsasl2-modules-gssapi-mit \
+      libpq-dev \
+      libecpg-dev \
+      libldap2-dev
+
+# Copy compiled things from previous stages
+COPY --from=superset-node /app/superset/static/assets superset/static/assets
+
+# Merging translations from backend and frontend stages
+COPY --from=superset-node /app/superset/translations superset/translations
+COPY --from=python-translation-compiler /app/translations_mo superset/translations
+
+HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
+CMD ["/app/docker/entrypoints/run-server.sh"]
+EXPOSE ${SUPERSET_PORT}
+
+######################################################################
+# Final lean image...
+######################################################################
+FROM python-common AS lean
+COPY superset superset
 
 # Install Python dependencies using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
+COPY requirements/base.txt requirements/
-    --mount=type=cache,target=/root/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh --requires-build-essential -r requirements/base.txt
+    /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt && \
+    uv pip install .
 
-# Copy the compiled frontend assets from the node image
+RUN python -m compileall /app/superset
-COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
 
-# Copy the main Superset source code
-COPY --chown=superset:superset superset superset
-
-# Install Superset itself using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
-    --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh -e .
-
-# Copy .json translations from the node image
-COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
-
-# Compile backend translations and clean up
-COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
-        ./scripts/translations/generate_mo_files.sh \
-        && chown -R superset:superset superset/translations; \
-    fi \
-    && rm -rf superset/translations/messages.pot \
-              superset/translations/*/LC_MESSAGES/*.po
-
-# Add server run script
-COPY --chmod=755 ./docker/run-server.sh /usr/bin/
-
-# Set user and healthcheck
 USER superset
-HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
-
-# Expose port and set CMD
-EXPOSE ${SUPERSET_PORT}
-CMD ["/usr/bin/run-server.sh"]
-
 
 ######################################################################
 # Dev image...
 ######################################################################
-FROM lean AS dev
+FROM python-common AS dev
+COPY superset superset
 
-USER root
+# Debian libs needed for dev
-
+RUN /app/docker/apt-install.sh \
-# Install dev dependencies
+    git \
-RUN --mount=type=bind,source=./docker,target=/docker \
+    pkg-config \
-    /docker/apt-install.sh \
+    default-libmysqlclient-dev
-        libnss3 \
-        libdbus-glib-1-2 \
-        libgtk-3-0 \
-        libx11-xcb1 \
-        libasound2 \
-        libxtst6 \
-        git \
-        pkg-config
-
-# Install Playwright and its dependencies
-RUN --mount=type=cache,target=/root/.cache/pip \
-    uv pip install --system playwright \
-    && playwright install-deps
-
-# Optionally install Chromium
-RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
-        playwright install chromium; \
-    else \
-        echo "Skipping Chromium installation in dev mode"; \
-    fi
-
-# Install GeckoDriver WebDriver and Firefox (if required)
-ARG GECKODRIVER_VERSION=v0.34.0
-ARG FIREFOX_VERSION=125.0.3
-RUN --mount=type=bind,source=./docker,target=/docker \
-    if [ "$INCLUDE_FIREFOX" = "true" ]; then \
-        /docker/apt-install.sh wget bzip2 \
-        && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
-        && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
-        && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-        && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
-    else \
-        echo "Skipping Firefox installation in dev mode"; \
-    fi
-
-# Install MySQL client dependencies
-RUN --mount=type=bind,source=./docker,target=/docker \
-    /docker/apt-install.sh default-libmysqlclient-dev
 
 # Copy development requirements and install them
-COPY --chown=superset:superset requirements/development.txt requirements/
+COPY requirements/*.txt requirements/
-RUN --mount=type=bind,source=./docker,target=/docker \
+# Install Python dependencies using docker/pip-install.sh
-    --mount=type=cache,target=/root/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh --requires-build-essential -r requirements/development.txt
+    /app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt && \
+    uv pip install .
+
+RUN python -m compileall /app/superset
 
 USER superset
 
@@ -232,6 +227,4 @@ USER superset
 ######################################################################
 FROM lean AS ci
 
-COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
+CMD ["/app/docker/entrypoints/docker-ci.sh"]
-
-CMD ["/app/docker/docker-ci.sh"]

UPDATING.md

@@ -30,6 +30,7 @@ assists people when migrating to a new version.
 - [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`.
 - [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before.
 - [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context.
+- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ...
 
 ### Potential Downtime
 

docker/.env

@@ -17,6 +17,7 @@
 
 
 COMPOSE_PROJECT_NAME=superset
+DEV_MODE=true
 
 # database configurations (do not modify)
 DATABASE_DB=superset

docker/docker-bootstrap.sh

@@ -18,6 +18,11 @@
 
 set -eo pipefail
 
+# Make python interactive
+if [ "$DEV_MODE" == "true" ]; then
+    echo "Reinstalling the app in editable mode"
+    uv pip install -e .
+fi
 REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
 # If Cypress run – overwrite the password for admin and export env variables
 if [ "$CYPRESS_CONFIG" == "true" ]; then

docker/pip-install.sh

@@ -47,10 +47,10 @@ fi
 # Choose whether to use pip cache
 if $USE_CACHE; then
   echo "Using pip cache..."
-  uv pip install --system "${ARGS[@]}"
+  uv pip install "${ARGS[@]}"
 else
   echo "Disabling pip cache..."
-  uv pip install --system --no-cache-dir "${ARGS[@]}"
+  uv pip install --no-cache-dir "${ARGS[@]}"
 fi
 
 # Remove build-essential if it was installed

requirements/translations.in

@@ -0,0 +1 @@
+babel

requirements/translations.txt

@@ -0,0 +1,9 @@
+# SHA1:cad160f3d4cd7c33896f42a479eeaa1b5bedc5fb
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+babel==2.16.0
+    # via -r requirements/translations.in