feat(docker): support running multiple Superset instances simultaneously (#36751)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

.envrc.example

@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Auto-configure Docker Compose for multi-instance support
+# Requires direnv: https://direnv.net/
+#
+# Install: brew install direnv (or apt install direnv)
+# Setup: Add 'eval "$(direnv hook bash)"' to ~/.bashrc (or ~/.zshrc)
+# Allow: Run 'direnv allow' in this directory once
+
+# Generate unique project name from directory
+export COMPOSE_PROJECT_NAME=$(basename "$PWD" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g')
+
+# Find available ports sequentially to avoid collisions
+_is_free() { ! lsof -i ":$1" &>/dev/null 2>&1; }
+
+_p=80;    while ! _is_free $_p; do ((_p++)); done; export NGINX_PORT=$_p
+_p=8088;  while ! _is_free $_p; do ((_p++)); done; export SUPERSET_PORT=$_p
+_p=9000;  while ! _is_free $_p; do ((_p++)); done; export NODE_PORT=$_p
+_p=8080;  while ! _is_free $_p || [ $_p -eq $NGINX_PORT ]; do ((_p++)); done; export WEBSOCKET_PORT=$_p
+_p=8081;  while ! _is_free $_p || [ $_p -eq $WEBSOCKET_PORT ]; do ((_p++)); done; export CYPRESS_PORT=$_p
+_p=5432;  while ! _is_free $_p; do ((_p++)); done; export DATABASE_PORT=$_p
+_p=6379;  while ! _is_free $_p; do ((_p++)); done; export REDIS_PORT=$_p
+
+unset _p _is_free
+
+echo "🐳 Superset configured: http://localhost:$SUPERSET_PORT (dev: localhost:$NODE_PORT)"

Makefile

@@ -18,7 +18,7 @@
 # Python version installed; we need 3.10-3.11
 PYTHON=`command -v python3.11 || command -v python3.10`
 
-.PHONY: install superset venv pre-commit
+.PHONY: install superset venv pre-commit up down logs ps nuke
 
 install: superset pre-commit
 
@@ -112,3 +112,22 @@ report-celery-beat:
 
 admin-user:
 	superset fab create-admin
+
+# Docker Compose with auto-assigned ports (for running multiple instances)
+up:
+	./scripts/docker-compose-up.sh
+
+up-detached:
+	./scripts/docker-compose-up.sh -d
+
+down:
+	./scripts/docker-compose-up.sh down
+
+logs:
+	./scripts/docker-compose-up.sh logs -f
+
+ps:
+	./scripts/docker-compose-up.sh ps
+
+nuke:
+	./scripts/docker-compose-up.sh nuke

docker-compose.yml

@@ -54,10 +54,9 @@ services:
       - path: docker/.env-local # optional override
         required: false
     image: nginx:latest
-    container_name: superset_nginx
     restart: unless-stopped
     ports:
-      - "80:80"
+      - "${NGINX_PORT:-80}:80"
     extra_hosts:
       - "host.docker.internal:host-gateway"
     volumes:
@@ -66,10 +65,9 @@ services:
 
   redis:
     image: redis:7
-    container_name: superset_cache
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      - "127.0.0.1:${REDIS_PORT:-6379}:6379"
     volumes:
       - redis:/data
 
@@ -80,10 +78,9 @@ services:
       - path: docker/.env-local # optional override
         required: false
     image: postgres:16
-    container_name: superset_db
     restart: unless-stopped
     ports:
-      - "127.0.0.1:5432:5432"
+      - "127.0.0.1:${DATABASE_PORT:-5432}:5432"
     volumes:
       - db_home:/var/lib/postgresql/data
       - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
@@ -96,13 +93,12 @@ services:
         required: false
     build:
       <<: *common-build
-    container_name: superset_app
     command: ["/app/docker/docker-bootstrap.sh", "app"]
     restart: unless-stopped
     ports:
-      - 8088:8088
+      - ${SUPERSET_PORT:-8088}:8088
       # When in cypress-mode ->
-      - 8081:8081
+      - ${CYPRESS_PORT:-8081}:8081
     extra_hosts:
       - "host.docker.internal:host-gateway"
     user: *superset-user
@@ -114,10 +110,9 @@ services:
       SUPERSET__SQLALCHEMY_EXAMPLES_URI: "duckdb:////app/data/examples.duckdb"
 
   superset-websocket:
-    container_name: superset_websocket
     build: ./superset-websocket
     ports:
-      - 8080:8080
+      - ${WEBSOCKET_PORT:-8080}:8080
     extra_hosts:
       - "host.docker.internal:host-gateway"
     depends_on:
@@ -149,7 +144,6 @@ services:
   superset-init:
     build:
       <<: *common-build
-    container_name: superset_init
     command: ["/app/docker/docker-init.sh"]
     env_file:
       - path: docker/.env # default
@@ -186,9 +180,10 @@ services:
       SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
       # configuring the dev-server to use the host.docker.internal to connect to the backend
       superset: "http://superset:8088"
+      # Bind to all interfaces so Docker port mapping works
+      WEBPACK_DEVSERVER_HOST: "0.0.0.0"
     ports:
-      - "127.0.0.1:9000:9000"  # exposing the dynamic webpack dev server
-    container_name: superset_node
+      - "127.0.0.1:${NODE_PORT:-9000}:9000"  # exposing the dynamic webpack dev server
     command: ["/app/docker/docker-frontend.sh"]
     env_file:
       - path: docker/.env # default
@@ -200,7 +195,6 @@ services:
   superset-worker:
     build:
       <<: *common-build
-    container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
     env_file:
       - path: docker/.env # default
@@ -226,7 +220,6 @@ services:
   superset-worker-beat:
     build:
       <<: *common-build
-    container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
     env_file:
       - path: docker/.env # default
@@ -244,7 +237,6 @@ services:
   superset-tests-worker:
     build:
       <<: *common-build
-    container_name: superset_tests_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
     env_file:
       - path: docker/.env # default

docker/.env

@@ -21,6 +21,15 @@ PYTHONUNBUFFERED=1
 COMPOSE_PROJECT_NAME=superset
 DEV_MODE=true
 
+# Port configuration (override in .env-local for multiple instances)
+# NGINX_PORT=80
+# SUPERSET_PORT=8088
+# NODE_PORT=9000
+# WEBSOCKET_PORT=8080
+# CYPRESS_PORT=8081
+# DATABASE_PORT=5432
+# REDIS_PORT=6379
+
 # database configurations (do not modify)
 DATABASE_DB=superset
 DATABASE_HOST=db

docker/.env-local.example

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------
+# Example .env-local file for running multiple Superset instances
+# Copy this file to .env-local and customize for your setup
+# -----------------------------------------------------------------------
+
+# Unique project name prevents container/volume conflicts between clones
+# Each clone should have a different name (e.g., superset-pr123, superset-feature-x)
+COMPOSE_PROJECT_NAME=superset-dev2
+
+# Port offsets for running multiple instances simultaneously
+# Instance 1 (default): 80, 8088, 9000, 8080, 8081, 5432, 6379
+# Instance 2 example:   81, 8089, 9001, 8082, 8083, 5433, 6380
+NGINX_PORT=81
+SUPERSET_PORT=8089
+NODE_PORT=9001
+WEBSOCKET_PORT=8082
+CYPRESS_PORT=8083
+DATABASE_PORT=5433
+REDIS_PORT=6380
+
+# For verbose logging during development:
+# SUPERSET_LOG_LEVEL=debug

docker/README.md

@@ -77,6 +77,34 @@ To run the container, simply run: `docker compose up`
 After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088)
 to start your journey.
 
+### Running Multiple Instances
+
+If you need to run multiple Superset instances simultaneously (e.g., different branches or clones), use the make targets which automatically find available ports:
+
+```bash
+make up
+```
+
+This automatically:
+- Generates a unique project name from your directory
+- Finds available ports (incrementing from defaults if in use)
+- Displays the assigned URLs before starting
+
+Available commands (run from repo root):
+
+| Command | Description |
+|---------|-------------|
+| `make up` | Start services (foreground) |
+| `make up-detached` | Start services (background) |
+| `make down` | Stop all services |
+| `make ps` | Show running containers |
+| `make logs` | Follow container logs |
+| `make nuke` | Stop, remove volumes & local images |
+
+From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up`
+
+**Important**: Always use these commands instead of plain `docker compose down`, which won't know the correct project name.
+
 ## Developing
 
 While running, the container server will reload on modification of the Superset Python and JavaScript source code.

docs/developer_portal/contributing/development-setup.md

@@ -139,6 +139,39 @@ docker volume rm superset_db_home
 docker-compose up
 ```
 
+### Running multiple instances
+
+If you need to run multiple Superset clones simultaneously (e.g., testing different branches),
+use `make up` instead of `docker compose up`:
+
+```bash
+make up
+```
+
+This automatically:
+- Generates a unique project name from your directory name
+- Finds available ports (incrementing from 8088, 9000, etc. if already in use)
+- Displays the assigned URLs before starting
+
+Each clone gets isolated containers and volumes, so you can run them side-by-side without conflicts.
+
+Available commands (run from repo root):
+
+| Command | Description |
+|---------|-------------|
+| `make up` | Start services (foreground) |
+| `make up-detached` | Start services (background) |
+| `make down` | Stop all services |
+| `make ps` | Show running containers |
+| `make logs` | Follow container logs |
+| `make nuke` | Stop, remove volumes & local images |
+
+From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up`
+
+:::warning
+Always use these commands instead of plain `docker compose down`, which won't know the correct project name for your instance.
+:::
+
 ## GitHub Codespaces (Cloud Development)
 
 GitHub Codespaces provides a complete, pre-configured development environment in the cloud. This is ideal for:

scripts/docker-compose-up.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------
+# Smart docker-compose wrapper for running multiple Superset instances
+#
+# Features:
+#   - Auto-generates unique project name from directory
+#   - Finds available ports automatically
+#   - No manual .env-local editing needed
+#
+# Usage:
+#   ./scripts/docker-compose-up.sh [docker-compose args...]
+#
+# Examples:
+#   ./scripts/docker-compose-up.sh           # Start all services
+#   ./scripts/docker-compose-up.sh -d        # Start detached
+#   ./scripts/docker-compose-up.sh down      # Stop services
+# -----------------------------------------------------------------------
+
+set -e
+
+# Get the repo root directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Generate project name from directory name (sanitized for Docker)
+DIR_NAME=$(basename "$REPO_ROOT")
+PROJECT_NAME=$(echo "$DIR_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
+
+# Function to check if a port is available
+is_port_available() {
+    local port=$1
+    if command -v lsof &> /dev/null; then
+        ! lsof -i ":$port" &> /dev/null
+    elif command -v netstat &> /dev/null; then
+        ! netstat -tuln 2>/dev/null | grep -q ":$port "
+    elif command -v ss &> /dev/null; then
+        ! ss -tuln 2>/dev/null | grep -q ":$port "
+    else
+        # If no tool available, assume port is available
+        return 0
+    fi
+}
+
+# Track ports we've already claimed in this session
+CLAIMED_PORTS=""
+
+# Function to check if port is available and not already claimed
+is_port_free() {
+    local port=$1
+    # Check not already claimed by us
+    if [[ " $CLAIMED_PORTS " =~ " $port " ]]; then
+        return 1
+    fi
+    # Check not in use on system
+    is_port_available $port
+}
+
+# Function to find and claim next available port starting from base
+# Sets the result in the variable named by $2
+find_and_claim_port() {
+    local base_port=$1
+    local var_name=$2
+    local max_attempts=100
+    local port=$base_port
+
+    for ((i=0; i<max_attempts; i++)); do
+        if is_port_free $port; then
+            CLAIMED_PORTS="$CLAIMED_PORTS $port"
+            eval "$var_name=$port"
+            return 0
+        fi
+        ((port++))
+    done
+
+    echo "ERROR: Could not find available port starting from $base_port" >&2
+    return 1
+}
+
+# Base ports (defaults from docker-compose.yml)
+BASE_NGINX=80
+BASE_SUPERSET=8088
+BASE_NODE=9000
+BASE_WEBSOCKET=8080
+BASE_CYPRESS=8081
+BASE_DATABASE=5432
+BASE_REDIS=6379
+
+# Find available ports (no subshells - claims persist correctly)
+echo "🔍 Finding available ports..."
+find_and_claim_port $BASE_NGINX NGINX_PORT
+find_and_claim_port $BASE_SUPERSET SUPERSET_PORT
+find_and_claim_port $BASE_NODE NODE_PORT
+find_and_claim_port $BASE_WEBSOCKET WEBSOCKET_PORT
+find_and_claim_port $BASE_CYPRESS CYPRESS_PORT
+find_and_claim_port $BASE_DATABASE DATABASE_PORT
+find_and_claim_port $BASE_REDIS REDIS_PORT
+
+# Export for docker-compose
+export COMPOSE_PROJECT_NAME="$PROJECT_NAME"
+export NGINX_PORT
+export SUPERSET_PORT
+export NODE_PORT
+export WEBSOCKET_PORT
+export CYPRESS_PORT
+export DATABASE_PORT
+export REDIS_PORT
+
+echo ""
+echo "🐳 Starting Superset with:"
+echo "   Project:    $PROJECT_NAME"
+echo "   Superset:   http://localhost:$SUPERSET_PORT"
+echo "   Dev Server: http://localhost:$NODE_PORT"
+echo "   Nginx:      http://localhost:$NGINX_PORT"
+echo "   WebSocket:  localhost:$WEBSOCKET_PORT"
+echo "   Database:   localhost:$DATABASE_PORT"
+echo "   Redis:      localhost:$REDIS_PORT"
+echo ""
+
+# Change to repo root
+cd "$REPO_ROOT"
+
+# Handle special commands
+case "${1:-}" in
+    --dry-run)
+        echo "✅ Dry run complete. To start, run without --dry-run"
+        exit 0
+        ;;
+    --env)
+        # Output as sourceable environment variables
+        echo "export COMPOSE_PROJECT_NAME='$PROJECT_NAME'"
+        echo "export NGINX_PORT=$NGINX_PORT"
+        echo "export SUPERSET_PORT=$SUPERSET_PORT"
+        echo "export NODE_PORT=$NODE_PORT"
+        echo "export WEBSOCKET_PORT=$WEBSOCKET_PORT"
+        echo "export CYPRESS_PORT=$CYPRESS_PORT"
+        echo "export DATABASE_PORT=$DATABASE_PORT"
+        echo "export REDIS_PORT=$REDIS_PORT"
+        exit 0
+        ;;
+    down|stop|logs|ps|exec|restart)
+        # Pass through to docker compose
+        docker compose "$@"
+        ;;
+    nuke)
+        # Nuclear option: remove everything (containers, volumes, local images)
+        echo "💥 Nuking all containers, volumes, and locally-built images for $PROJECT_NAME..."
+        docker compose down -v --rmi local
+        echo "✅ Done. Run 'make up' or './scripts/docker-compose-up.sh' to start fresh."
+        ;;
+    *)
+        # Default: start services
+        docker compose up "$@"
+        ;;
+esac