feat(docker): Add pytest support to docker-compose-light.yml (#34373)

Co-authored-by: Claude <noreply@anthropic.com>
2026-04-19 16:14:52 +00:00 · 2025-08-06 00:17:50 -04:00
parent 6f5d9c989a
commit 246181a546
15 changed files with 687 additions and 70 deletions
--- a/docker/docker-pytest-entrypoint.sh
+++ b/docker/docker-pytest-entrypoint.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# Wait for PostgreSQL to be ready
+echo "Waiting for database to be ready..."
+for i in {1..30}; do
+  if python3 -c "
+import psycopg2
+try:
+    conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='superset_light')
+    conn.close()
+    print('Database is ready!')
+except:
+    exit(1)
+" 2>/dev/null; then
+    echo "Database connection established!"
+    break
+  fi
+  echo "Waiting for database... ($i/30)"
+  if [ $i -eq 30 ]; then
+    echo "Database connection timeout after 30 seconds"
+    exit 1
+  fi
+  sleep 1
+done
+
+# Handle database setup based on FORCE_RELOAD
+if [ "${FORCE_RELOAD}" = "true" ]; then
+  echo "Force reload requested - resetting test database"
+  # Drop and recreate the test database using Python
+  python3 -c "
+import psycopg2
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+# Connect to default database
+conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='superset_light')
+conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+cur = conn.cursor()
+
+# Drop and recreate test database
+try:
+    cur.execute('DROP DATABASE IF EXISTS test')
+except:
+    pass
+
+cur.execute('CREATE DATABASE test')
+conn.close()
+
+# Connect to test database to create schemas
+conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='test')
+conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+cur = conn.cursor()
+
+cur.execute('CREATE SCHEMA sqllab_test_db')
+cur.execute('CREATE SCHEMA admin_database')
+
+cur.close()
+conn.close()
+print('Test database reset successfully')
+"
+  # Use --no-reset-db since we already reset it
+  FLAGS="--no-reset-db"
+else
+  echo "Using existing test database (set FORCE_RELOAD=true to reset)"
+  FLAGS="--no-reset-db"
+
+  # Ensure test database exists using Python
+  python3 -c "
+import psycopg2
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+# Check if test database exists
+try:
+    conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='test')
+    conn.close()
+    print('Test database already exists')
+except:
+    print('Creating test database...')
+    # Connect to default database to create test database
+    conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='superset_light')
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = conn.cursor()
+
+    # Create test database
+    cur.execute('CREATE DATABASE test')
+    conn.close()
+
+    # Connect to test database to create schemas
+    conn = psycopg2.connect(host='db-light', user='superset', password='superset', database='test')
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = conn.cursor()
+
+    cur.execute('CREATE SCHEMA IF NOT EXISTS sqllab_test_db')
+    cur.execute('CREATE SCHEMA IF NOT EXISTS admin_database')
+
+    cur.close()
+    conn.close()
+    print('Test database created successfully')
+"
+fi
+
+# Always run database migrations to ensure schema is up to date
+echo "Running database migrations..."
+cd /app
+superset db upgrade
+
+# Initialize test environment if needed
+if [ "${FORCE_RELOAD}" = "true" ] || [ ! -f "/app/superset_home/.test_initialized" ]; then
+  echo "Initializing test environment..."
+  # Run initialization commands
+  superset init
+  echo "Loading test users..."
+  superset load-test-users
+
+  # Mark as initialized
+  touch /app/superset_home/.test_initialized
+else
+  echo "Test environment already initialized (skipping init and load-test-users)"
+  echo "Tip: Use FORCE_RELOAD=true to reinitialize the test database"
+fi
+
+# Create missing scripts needed for tests
+if [ ! -f "/app/scripts/tag_latest_release.sh" ]; then
+  echo "Creating missing tag_latest_release.sh script for tests..."
+  cp /app/docker/tag_latest_release.sh /app/scripts/tag_latest_release.sh 2>/dev/null || true
+fi
+
+# Install pip module for Shillelagh compatibility (aligns with CI environment)
+echo "Installing pip module for Shillelagh compatibility..."
+uv pip install pip
+
+# If arguments provided, execute them
+if [ $# -gt 0 ]; then
+  exec "$@"
+fi
--- a/docker/pythonpath_dev/superset_test_config_light.py
+++ b/docker/pythonpath_dev/superset_test_config_light.py
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Test configuration for docker-compose-light.yml - uses SimpleCache instead of Redis
+
+# Import all settings from the main test config first
+import os
+import sys
+
+# Add the tests directory to the path to import the test config
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+from tests.integration_tests.superset_test_config import *  # noqa: F403
+
+# Override Redis-based caching to use simple in-memory cache
+CACHE_CONFIG = {
+    "CACHE_TYPE": "SimpleCache",
+    "CACHE_DEFAULT_TIMEOUT": 300,
+    "CACHE_KEY_PREFIX": "superset_test_",
+}
+
+DATA_CACHE_CONFIG = {
+    **CACHE_CONFIG,
+    "CACHE_DEFAULT_TIMEOUT": 30,
+    "CACHE_KEY_PREFIX": "superset_test_data_",
+}
+
+# Keep SimpleCache for these as they're already using it
+# FILTER_STATE_CACHE_CONFIG - already SimpleCache in parent
+# EXPLORE_FORM_DATA_CACHE_CONFIG - already SimpleCache in parent
+
+# Disable Celery for lightweight testing
+CELERY_CONFIG = None
+
+# Use FileSystemCache for SQL Lab results instead of Redis
+from flask_caching.backends.filesystemcache import FileSystemCache  # noqa: E402
+
+RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab_test")
+
+# Override WEBDRIVER_BASEURL for tests to match expected values
+WEBDRIVER_BASEURL = "http://0.0.0.0:8080/"
+WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
--- a/docker/tag_latest_release.sh
+++ b/docker/tag_latest_release.sh
@@ -0,0 +1,190 @@
+#! /bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+run_git_tag () {
+  if [[ "$DRY_RUN" == "false" ]] && [[ "$SKIP_TAG" == "false" ]]
+  then
+    git tag -a -f latest "${GITHUB_TAG_NAME}" -m "latest tag"
+    echo "${GITHUB_TAG_NAME} has been tagged 'latest'"
+  fi
+  exit 0
+}
+
+###
+# separating out git commands into functions so they can be mocked in unit tests
+###
+git_show_ref () {
+  if [[ "$TEST_ENV" == "true" ]]
+  then
+    if [[ "$GITHUB_TAG_NAME" == "does_not_exist" ]]
+        # mock return for testing only
+    then
+      echo ""
+    else
+      echo "2817aebd69dc7d199ec45d973a2079f35e5658b6 refs/tags/${GITHUB_TAG_NAME}"
+    fi
+  fi
+  result=$(git show-ref "${GITHUB_TAG_NAME}")
+  echo "${result}"
+}
+
+get_latest_tag_list () {
+  if [[ "$TEST_ENV" == "true" ]]
+  then
+    echo "(tag: 2.1.0, apache/2.1test)"
+  else
+    result=$(git show-ref --tags --dereference latest | awk '{print $2}' | xargs git show --pretty=tformat:%d -s | grep tag:)
+    echo "${result}"
+  fi
+}
+###
+
+split_string () {
+  local version="$1"
+  local delimiter="$2"
+  local components=()
+  local tmp=""
+  for (( i=0; i<${#version}; i++ )); do
+    local char="${version:$i:1}"
+    if [[ "$char" != "$delimiter" ]]; then
+      tmp="$tmp$char"
+    elif [[ -n "$tmp" ]]; then
+      components+=("$tmp")
+      tmp=""
+    fi
+  done
+  if [[ -n "$tmp" ]]; then
+    components+=("$tmp")
+  fi
+  echo "${components[@]}"
+}
+
+DRY_RUN=false
+
+# get params passed in with script when it was run
+# --dry-run is optional and returns the value of SKIP_TAG, but does not run the git tag statement
+# A tag name is required as a param. A SHA won't work. You must first tag a sha with a release number
+# and then run this script
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    --dry-run)
+    DRY_RUN=true
+    shift # past value
+    ;;
+    *)    # this should be the tag name
+    GITHUB_TAG_NAME=$key
+    shift # past value
+    ;;
+esac
+done
+
+if [ -z "${GITHUB_TAG_NAME}" ]; then
+    echo "Missing tag parameter, usage: ./scripts/tag_latest_release.sh <GITHUB_TAG_NAME>"
+    echo "SKIP_TAG=true" >> $GITHUB_OUTPUT
+    exit 1
+fi
+
+if [ -z "$(git_show_ref)" ]; then
+    echo "The tag ${GITHUB_TAG_NAME} does not exist. Please use a different tag."
+    echo "SKIP_TAG=true" >> $GITHUB_OUTPUT
+    exit 0
+fi
+
+# check that this tag only contains a proper semantic version
+if ! [[ ${GITHUB_TAG_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+then
+  echo "This tag ${GITHUB_TAG_NAME} is not a valid release version. Not tagging."
+  echo "SKIP_TAG=true" >> $GITHUB_OUTPUT
+  exit 1
+fi
+
+## split the current GITHUB_TAG_NAME into an array at the dot
+THIS_TAG_NAME=$(split_string "${GITHUB_TAG_NAME}" ".")
+
+# look up the 'latest' tag on git
+LATEST_TAG_LIST=$(get_latest_tag_list) || echo 'not found'
+
+# if 'latest' tag doesn't exist, then set this commit to latest
+if [[ -z "$LATEST_TAG_LIST" ]]
+then
+  echo "there are no latest tags yet, so I'm going to start by tagging this sha as the latest"
+  run_git_tag
+  exit 0
+fi
+
+# remove parenthesis and tag: from the list of tags
+LATEST_TAGS_STRINGS=$(echo "$LATEST_TAG_LIST" | sed 's/tag: \([^,]*\)/\1/g' | tr -d '()')
+
+LATEST_TAGS=$(split_string "$LATEST_TAGS_STRINGS" ",")
+TAGS=($(split_string "$LATEST_TAGS" " "))
+
+# Initialize a flag for comparison result
+compare_result=""
+
+# Iterate through the tags of the latest release
+for tag in $TAGS
+do
+  if [[ $tag == "latest" ]]; then
+    continue
+  else
+    ## extract just the version from this tag
+    LATEST_RELEASE_TAG="$tag"
+    echo "LATEST_RELEASE_TAG: ${LATEST_RELEASE_TAG}"
+
+    # check that this only contains a proper semantic version
+    if ! [[ ${LATEST_RELEASE_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+    then
+      echo "'Latest' has been associated with tag ${LATEST_RELEASE_TAG} which is not a valid release version. Looking for another."
+      continue
+    fi
+    echo "The current release with the latest tag is version ${LATEST_RELEASE_TAG}"
+    # Split the version strings into arrays
+    THIS_TAG_NAME_ARRAY=($(split_string "$THIS_TAG_NAME" "."))
+    LATEST_RELEASE_TAG_ARRAY=($(split_string "$LATEST_RELEASE_TAG" "."))
+
+    # Iterate through the components of the version strings
+    for (( j=0; j<${#THIS_TAG_NAME_ARRAY[@]}; j++ )); do
+        echo "Comparing ${THIS_TAG_NAME_ARRAY[$j]} to ${LATEST_RELEASE_TAG_ARRAY[$j]}"
+        if [[ $((THIS_TAG_NAME_ARRAY[$j])) > $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+            compare_result="greater"
+            break
+        elif [[ $((THIS_TAG_NAME_ARRAY[$j])) < $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+            compare_result="lesser"
+            break
+        fi
+    done
+  fi
+done
+
+# Determine the result based on the comparison
+if [[ -z "$compare_result" ]]; then
+    echo "Versions are equal"
+    echo "SKIP_TAG=true" >> $GITHUB_OUTPUT
+elif [[ "$compare_result" == "greater" ]]; then
+    echo "This release tag ${GITHUB_TAG_NAME} is newer than the latest."
+    echo "SKIP_TAG=false" >> $GITHUB_OUTPUT
+    # Add other actions you want to perform for a newer version
+elif [[ "$compare_result" == "lesser" ]]; then
+    echo "This release tag ${GITHUB_TAG_NAME} is older than the latest."
+    echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging."
+    # if you've gotten this far, then we don't want to run any tags in the next step
+    echo "SKIP_TAG=true" >> $GITHUB_OUTPUT
+fi