diff --git a/.envrc.example b/.envrc.example new file mode 100644 index 00000000000..1d1f80ff3f2 --- /dev/null +++ b/.envrc.example @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Auto-configure Docker Compose for multi-instance support +# Requires direnv: https://direnv.net/ +# +# Install: brew install direnv (or apt install direnv) +# Setup: Add 'eval "$(direnv hook bash)"' to ~/.bashrc (or ~/.zshrc) +# Allow: Run 'direnv allow' in this directory once + +# Generate unique project name from directory +export COMPOSE_PROJECT_NAME=$(basename "$PWD" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g') + +# Find available ports sequentially to avoid collisions +_is_free() { ! lsof -i ":$1" &>/dev/null 2>&1; } + +_p=80; while ! _is_free $_p; do ((_p++)); done; export NGINX_PORT=$_p +_p=8088; while ! _is_free $_p; do ((_p++)); done; export SUPERSET_PORT=$_p +_p=9000; while ! _is_free $_p; do ((_p++)); done; export NODE_PORT=$_p +_p=8080; while ! _is_free $_p || [ $_p -eq $NGINX_PORT ]; do ((_p++)); done; export WEBSOCKET_PORT=$_p +_p=8081; while ! _is_free $_p || [ $_p -eq $WEBSOCKET_PORT ]; do ((_p++)); done; export CYPRESS_PORT=$_p +_p=5432; while ! _is_free $_p; do ((_p++)); done; export DATABASE_PORT=$_p +_p=6379; while ! _is_free $_p; do ((_p++)); done; export REDIS_PORT=$_p + +unset _p _is_free + +echo "🐳 Superset configured: http://localhost:$SUPERSET_PORT (dev: localhost:$NODE_PORT)" diff --git a/Makefile b/Makefile index 4a7121fd34f..9d0a73c9e7e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Python version installed; we need 3.10-3.11 PYTHON=`command -v python3.11 || command -v python3.10` -.PHONY: install superset venv pre-commit +.PHONY: install superset venv pre-commit up down logs ps nuke install: superset pre-commit @@ -112,3 +112,22 @@ report-celery-beat: admin-user: superset fab create-admin + +# Docker Compose with auto-assigned ports (for running multiple instances) +up: + ./scripts/docker-compose-up.sh + +up-detached: + ./scripts/docker-compose-up.sh -d + +down: + ./scripts/docker-compose-up.sh down + +logs: + ./scripts/docker-compose-up.sh logs -f + +ps: + ./scripts/docker-compose-up.sh ps + +nuke: + ./scripts/docker-compose-up.sh nuke diff --git a/docker-compose.yml b/docker-compose.yml index 050c6b22ef3..5930951776f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,10 +54,9 @@ services: - path: docker/.env-local # optional override required: false image: nginx:latest - container_name: superset_nginx restart: unless-stopped ports: - - "80:80" + - "${NGINX_PORT:-80}:80" extra_hosts: - "host.docker.internal:host-gateway" volumes: @@ -66,10 +65,9 @@ services: redis: image: redis:7 - container_name: superset_cache restart: unless-stopped ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:${REDIS_PORT:-6379}:6379" volumes: - redis:/data @@ -80,10 +78,9 @@ services: - path: docker/.env-local # optional override required: false image: postgres:16 - container_name: superset_db restart: unless-stopped ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:${DATABASE_PORT:-5432}:5432" volumes: - db_home:/var/lib/postgresql/data - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d @@ -96,13 +93,12 @@ services: required: false build: <<: *common-build - container_name: superset_app command: ["/app/docker/docker-bootstrap.sh", "app"] restart: unless-stopped ports: - - 8088:8088 + - ${SUPERSET_PORT:-8088}:8088 # When in cypress-mode -> - - 8081:8081 + - ${CYPRESS_PORT:-8081}:8081 extra_hosts: - "host.docker.internal:host-gateway" user: *superset-user @@ -114,10 +110,9 @@ services: SUPERSET__SQLALCHEMY_EXAMPLES_URI: "duckdb:////app/data/examples.duckdb" superset-websocket: - container_name: superset_websocket build: ./superset-websocket ports: - - 8080:8080 + - ${WEBSOCKET_PORT:-8080}:8080 extra_hosts: - "host.docker.internal:host-gateway" depends_on: @@ -149,7 +144,6 @@ services: superset-init: build: <<: *common-build - container_name: superset_init command: ["/app/docker/docker-init.sh"] env_file: - path: docker/.env # default @@ -186,9 +180,10 @@ services: SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}" # configuring the dev-server to use the host.docker.internal to connect to the backend superset: "http://superset:8088" + # Bind to all interfaces so Docker port mapping works + WEBPACK_DEVSERVER_HOST: "0.0.0.0" ports: - - "127.0.0.1:9000:9000" # exposing the dynamic webpack dev server - container_name: superset_node + - "127.0.0.1:${NODE_PORT:-9000}:9000" # exposing the dynamic webpack dev server command: ["/app/docker/docker-frontend.sh"] env_file: - path: docker/.env # default @@ -200,7 +195,6 @@ services: superset-worker: build: <<: *common-build - container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - path: docker/.env # default @@ -226,7 +220,6 @@ services: superset-worker-beat: build: <<: *common-build - container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - path: docker/.env # default @@ -244,7 +237,6 @@ services: superset-tests-worker: build: <<: *common-build - container_name: superset_tests_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - path: docker/.env # default diff --git a/docker/.env b/docker/.env index a0cbb47deb5..def1be0d25d 100644 --- a/docker/.env +++ b/docker/.env @@ -21,6 +21,15 @@ PYTHONUNBUFFERED=1 COMPOSE_PROJECT_NAME=superset DEV_MODE=true +# Port configuration (override in .env-local for multiple instances) +# NGINX_PORT=80 +# SUPERSET_PORT=8088 +# NODE_PORT=9000 +# WEBSOCKET_PORT=8080 +# CYPRESS_PORT=8081 +# DATABASE_PORT=5432 +# REDIS_PORT=6379 + # database configurations (do not modify) DATABASE_DB=superset DATABASE_HOST=db diff --git a/docker/.env-local.example b/docker/.env-local.example new file mode 100644 index 00000000000..433c3160919 --- /dev/null +++ b/docker/.env-local.example @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------- +# Example .env-local file for running multiple Superset instances +# Copy this file to .env-local and customize for your setup +# ----------------------------------------------------------------------- + +# Unique project name prevents container/volume conflicts between clones +# Each clone should have a different name (e.g., superset-pr123, superset-feature-x) +COMPOSE_PROJECT_NAME=superset-dev2 + +# Port offsets for running multiple instances simultaneously +# Instance 1 (default): 80, 8088, 9000, 8080, 8081, 5432, 6379 +# Instance 2 example: 81, 8089, 9001, 8082, 8083, 5433, 6380 +NGINX_PORT=81 +SUPERSET_PORT=8089 +NODE_PORT=9001 +WEBSOCKET_PORT=8082 +CYPRESS_PORT=8083 +DATABASE_PORT=5433 +REDIS_PORT=6380 + +# For verbose logging during development: +# SUPERSET_LOG_LEVEL=debug diff --git a/docker/README.md b/docker/README.md index eeb2a8f3590..1ef46aa13ae 100644 --- a/docker/README.md +++ b/docker/README.md @@ -77,6 +77,34 @@ To run the container, simply run: `docker compose up` After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088) to start your journey. +### Running Multiple Instances + +If you need to run multiple Superset instances simultaneously (e.g., different branches or clones), use the make targets which automatically find available ports: + +```bash +make up +``` + +This automatically: +- Generates a unique project name from your directory +- Finds available ports (incrementing from defaults if in use) +- Displays the assigned URLs before starting + +Available commands (run from repo root): + +| Command | Description | +|---------|-------------| +| `make up` | Start services (foreground) | +| `make up-detached` | Start services (background) | +| `make down` | Stop all services | +| `make ps` | Show running containers | +| `make logs` | Follow container logs | +| `make nuke` | Stop, remove volumes & local images | + +From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up` + +**Important**: Always use these commands instead of plain `docker compose down`, which won't know the correct project name. + ## Developing While running, the container server will reload on modification of the Superset Python and JavaScript source code. diff --git a/docs/developer_portal/contributing/development-setup.md b/docs/developer_portal/contributing/development-setup.md index a25cca8dbc9..464c70a73c1 100644 --- a/docs/developer_portal/contributing/development-setup.md +++ b/docs/developer_portal/contributing/development-setup.md @@ -139,6 +139,39 @@ docker volume rm superset_db_home docker-compose up ``` +### Running multiple instances + +If you need to run multiple Superset clones simultaneously (e.g., testing different branches), +use `make up` instead of `docker compose up`: + +```bash +make up +``` + +This automatically: +- Generates a unique project name from your directory name +- Finds available ports (incrementing from 8088, 9000, etc. if already in use) +- Displays the assigned URLs before starting + +Each clone gets isolated containers and volumes, so you can run them side-by-side without conflicts. + +Available commands (run from repo root): + +| Command | Description | +|---------|-------------| +| `make up` | Start services (foreground) | +| `make up-detached` | Start services (background) | +| `make down` | Stop all services | +| `make ps` | Show running containers | +| `make logs` | Follow container logs | +| `make nuke` | Stop, remove volumes & local images | + +From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up` + +:::warning +Always use these commands instead of plain `docker compose down`, which won't know the correct project name for your instance. +::: + ## GitHub Codespaces (Cloud Development) GitHub Codespaces provides a complete, pre-configured development environment in the cloud. This is ideal for: diff --git a/scripts/docker-compose-up.sh b/scripts/docker-compose-up.sh new file mode 100755 index 00000000000..525d07e3c2d --- /dev/null +++ b/scripts/docker-compose-up.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------- +# Smart docker-compose wrapper for running multiple Superset instances +# +# Features: +# - Auto-generates unique project name from directory +# - Finds available ports automatically +# - No manual .env-local editing needed +# +# Usage: +# ./scripts/docker-compose-up.sh [docker-compose args...] +# +# Examples: +# ./scripts/docker-compose-up.sh # Start all services +# ./scripts/docker-compose-up.sh -d # Start detached +# ./scripts/docker-compose-up.sh down # Stop services +# ----------------------------------------------------------------------- + +set -e + +# Get the repo root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Generate project name from directory name (sanitized for Docker) +DIR_NAME=$(basename "$REPO_ROOT") +PROJECT_NAME=$(echo "$DIR_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//') + +# Function to check if a port is available +is_port_available() { + local port=$1 + if command -v lsof &> /dev/null; then + ! lsof -i ":$port" &> /dev/null + elif command -v netstat &> /dev/null; then + ! netstat -tuln 2>/dev/null | grep -q ":$port " + elif command -v ss &> /dev/null; then + ! ss -tuln 2>/dev/null | grep -q ":$port " + else + # If no tool available, assume port is available + return 0 + fi +} + +# Track ports we've already claimed in this session +CLAIMED_PORTS="" + +# Function to check if port is available and not already claimed +is_port_free() { + local port=$1 + # Check not already claimed by us + if [[ " $CLAIMED_PORTS " =~ " $port " ]]; then + return 1 + fi + # Check not in use on system + is_port_available $port +} + +# Function to find and claim next available port starting from base +# Sets the result in the variable named by $2 +find_and_claim_port() { + local base_port=$1 + local var_name=$2 + local max_attempts=100 + local port=$base_port + + for ((i=0; i&2 + return 1 +} + +# Base ports (defaults from docker-compose.yml) +BASE_NGINX=80 +BASE_SUPERSET=8088 +BASE_NODE=9000 +BASE_WEBSOCKET=8080 +BASE_CYPRESS=8081 +BASE_DATABASE=5432 +BASE_REDIS=6379 + +# Find available ports (no subshells - claims persist correctly) +echo "🔍 Finding available ports..." +find_and_claim_port $BASE_NGINX NGINX_PORT +find_and_claim_port $BASE_SUPERSET SUPERSET_PORT +find_and_claim_port $BASE_NODE NODE_PORT +find_and_claim_port $BASE_WEBSOCKET WEBSOCKET_PORT +find_and_claim_port $BASE_CYPRESS CYPRESS_PORT +find_and_claim_port $BASE_DATABASE DATABASE_PORT +find_and_claim_port $BASE_REDIS REDIS_PORT + +# Export for docker-compose +export COMPOSE_PROJECT_NAME="$PROJECT_NAME" +export NGINX_PORT +export SUPERSET_PORT +export NODE_PORT +export WEBSOCKET_PORT +export CYPRESS_PORT +export DATABASE_PORT +export REDIS_PORT + +echo "" +echo "🐳 Starting Superset with:" +echo " Project: $PROJECT_NAME" +echo " Superset: http://localhost:$SUPERSET_PORT" +echo " Dev Server: http://localhost:$NODE_PORT" +echo " Nginx: http://localhost:$NGINX_PORT" +echo " WebSocket: localhost:$WEBSOCKET_PORT" +echo " Database: localhost:$DATABASE_PORT" +echo " Redis: localhost:$REDIS_PORT" +echo "" + +# Change to repo root +cd "$REPO_ROOT" + +# Handle special commands +case "${1:-}" in + --dry-run) + echo "✅ Dry run complete. To start, run without --dry-run" + exit 0 + ;; + --env) + # Output as sourceable environment variables + echo "export COMPOSE_PROJECT_NAME='$PROJECT_NAME'" + echo "export NGINX_PORT=$NGINX_PORT" + echo "export SUPERSET_PORT=$SUPERSET_PORT" + echo "export NODE_PORT=$NODE_PORT" + echo "export WEBSOCKET_PORT=$WEBSOCKET_PORT" + echo "export CYPRESS_PORT=$CYPRESS_PORT" + echo "export DATABASE_PORT=$DATABASE_PORT" + echo "export REDIS_PORT=$REDIS_PORT" + exit 0 + ;; + down|stop|logs|ps|exec|restart) + # Pass through to docker compose + docker compose "$@" + ;; + nuke) + # Nuclear option: remove everything (containers, volumes, local images) + echo "💥 Nuking all containers, volumes, and locally-built images for $PROJECT_NAME..." + docker compose down -v --rmi local + echo "✅ Done. Run 'make up' or './scripts/docker-compose-up.sh' to start fresh." + ;; + *) + # Default: start services + docker compose up "$@" + ;; +esac