Extension loading

docker-compose-light.yml

@@ -67,6 +67,7 @@ x-superset-volumes: &superset-volumes
   - ./superset-frontend:/app/superset-frontend
   - superset_home_light:/app/superset_home
   - ./tests:/app/tests
+  - ./extensions:/app/extensions
 x-common-build: &common-build
   context: .
   target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`

docker/pythonpath_dev/superset_config.py

@@ -105,7 +105,15 @@ class CeleryConfig:
 
 CELERY_CONFIG = CeleryConfig
 
-FEATURE_FLAGS = {"ALERT_REPORTS": True}
+# Extensions configuration
+# For local development, point to the extensions directory
+# Note: If running in Docker, this path needs to be accessible from inside the container
+EXTENSIONS_PATH = os.getenv("EXTENSIONS_PATH", "/app/extensions")
+
+FEATURE_FLAGS = {
+    "ALERT_REPORTS": True,
+    "ENABLE_EXTENSIONS": True,
+}
 ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
 WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/"  # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/  # noqa: E501
 # The base URL for the email report hyperlinks.

extensions/requirements-snowflake.txt

@@ -0,0 +1,5 @@
+# Requirements for the Snowflake Semantic Layer extension
+# Install with: pip install -r extensions/requirements-snowflake.txt
+
+snowflake-connector-python>=3.0.0
+snowflake-sqlalchemy>=1.5.0

setup.py

@@ -30,9 +30,7 @@ with open(PACKAGE_JSON) as package_file:
 
 def get_git_sha() -> str:
     try:
-        output = subprocess.check_output(
-            ["git", "rev-parse", "HEAD"]
-        )  # noqa: S603, S607
+        output = subprocess.check_output(["git", "rev-parse", "HEAD"])  # noqa: S603, S607
         return output.decode().strip()
     except Exception:  # pylint: disable=broad-except
         return ""
@@ -60,9 +58,6 @@ setup(
     include_package_data=True,
     zip_safe=False,
     entry_points={
-        "superset.semantic_layers": [
-            "snowflake = superset.semantic_layers.snowflake:SnowflakeSemanticLayer"
-        ],
         "console_scripts": ["superset=superset.cli.main:superset"],
         # the `postgres` and `postgres+psycopg2://` schemes were removed in SQLAlchemy 1.4  # noqa: E501
         # add an alias here to prevent breaking existing databases

superset-snowflake-semantic-layer/README.md

@@ -0,0 +1,60 @@
+# Snowflake Semantic Layer Extension for Apache Superset
+
+This extension adds support for Snowflake Semantic Views to Apache Superset.
+
+## Installation
+
+### As a pip package (recommended for production)
+
+```bash
+pip install superset-snowflake-semantic-layer
+```
+
+This automatically installs all required dependencies.
+
+### As a Superset extension (.supx bundle)
+
+1. **Install dependencies** (the .supx bundle doesn't include pip packages):
+   ```bash
+   pip install snowflake-connector-python>=3.0.0 snowflake-sqlalchemy>=1.5.0
+   ```
+
+2. Build the extension bundle:
+   ```bash
+   cd superset-snowflake-semantic-layer
+   superset-extensions bundle
+   ```
+
+3. Copy the generated `.supx` file to your Superset extensions directory.
+
+4. Configure Superset to load extensions:
+   ```python
+   # superset_config.py
+   EXTENSIONS_PATH = "/path/to/extensions"
+   ```
+
+5. Restart Superset.
+
+## Configuration
+
+When adding a Snowflake Semantic Layer in Superset, you'll need to provide:
+
+- **Account Identifier**: Your Snowflake account identifier (e.g., `abc12345`)
+- **Authentication**: Either username/password or private key authentication
+- **Role** (optional): The default Snowflake role to use
+- **Warehouse** (optional): The default warehouse to use
+- **Database** (optional): The default database containing semantic views
+- **Schema** (optional): The default schema containing semantic views
+
+## Features
+
+- Browse and query Snowflake Semantic Views
+- Support for dimensions and metrics defined in semantic views
+- Filtering and aggregation through the Superset UI
+- Group limiting (top N) with optional "Other" grouping
+
+## Requirements
+
+- Apache Superset 4.0+
+- Snowflake account with semantic views
+- `snowflake-connector-python` >= 3.0.0

superset-snowflake-semantic-layer/backend/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "superset-snowflake-semantic-layer"
+version = "1.0.0"
+description = "Snowflake Semantic Layer extension for Apache Superset"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = [
+    "apache-superset",
+    "snowflake-connector-python>=3.0.0",
+    "snowflake-sqlalchemy>=1.5.0",
+    "pydantic>=2.0.0",
+    "cryptography>=41.0.0",
+]
+
+[project.entry-points."superset.semantic_layers"]
+snowflake = "snowflake_semantic_layer:SnowflakeSemanticLayer"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/snowflake_semantic_layer"]

superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/__init__.py

@@ -15,9 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration
-from superset.semantic_layers.snowflake.semantic_layer import SnowflakeSemanticLayer
-from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView
+from snowflake_semantic_layer.schemas import SnowflakeConfiguration
+from snowflake_semantic_layer.semantic_layer import SnowflakeSemanticLayer
+from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView
 
 __all__ = [
     "SnowflakeConfiguration",

superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_layer.py

@@ -24,9 +24,9 @@ from pydantic import create_model, Field
 from snowflake.connector import connect
 from snowflake.connector.connection import SnowflakeConnection
 
-from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration
-from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView
-from superset.semantic_layers.snowflake.utils import get_connection_parameters
+from snowflake_semantic_layer.schemas import SnowflakeConfiguration
+from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView
+from snowflake_semantic_layer.utils import get_connection_parameters
 from superset.semantic_layers.types import (
     SemanticLayerImplementation,
 )
@@ -210,10 +210,7 @@ class SnowflakeSemanticLayer(
         """
         Get the semantic views available in the semantic layer.
         """
-        # Avoid circular import
-        from superset.semantic_layers.snowflake.semantic_view import (
-            SnowflakeSemanticView,
-        )
+        from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView
 
         # create a new configuration with the runtime parameters
         configuration = self.configuration.model_copy(update=runtime_configuration)
@@ -242,10 +239,7 @@ class SnowflakeSemanticLayer(
         """
         Get a specific semantic view by name.
         """
-        # Avoid circular import
-        from superset.semantic_layers.snowflake.semantic_view import (
-            SnowflakeSemanticView,
-        )
+        from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView
 
         # create a new configuration with the additional parameters
         configuration = self.configuration.model_copy(update=additional_configuration)

superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_view.py

@@ -28,8 +28,8 @@ from pandas import DataFrame
 from snowflake.connector import connect, DictCursor
 from snowflake.sqlalchemy.snowdialect import SnowflakeDialect
 
-from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration
-from superset.semantic_layers.snowflake.utils import (
+from snowflake_semantic_layer.schemas import SnowflakeConfiguration
+from snowflake_semantic_layer.utils import (
     get_connection_parameters,
     substitute_parameters,
     validate_order_by,
@@ -515,7 +515,10 @@ class SnowflakeSemanticView(SemanticViewImplementation):
         # Check if temporal dimension is already in the order
         if order:
             for element, _ in order:
-                if isinstance(element, Dimension) and element.id == temporal_dimension.id:
+                if (
+                    isinstance(element, Dimension)
+                    and element.id == temporal_dimension.id
+                ):
                     return order
             # Prepend temporal dimension to existing order
             return [(temporal_dimension, OrderDirection.ASC)] + list(order)

superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/utils.py

@@ -24,12 +24,12 @@ from typing import Any, Sequence
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 
-from superset.exceptions import SupersetParseError
-from superset.semantic_layers.snowflake.schemas import (
+from snowflake_semantic_layer.schemas import (
     PrivateKeyAuth,
     SnowflakeConfiguration,
     UserPasswordAuth,
 )
+from superset.exceptions import SupersetParseError
 from superset.sql.parse import SQLStatement
 
 

superset-snowflake-semantic-layer/extension.json

@@ -0,0 +1,15 @@
+{
+  "id": "superset-snowflake-semantic-layer",
+  "name": "Snowflake Semantic Layer",
+  "version": "1.0.0",
+  "description": "Connect to semantic views stored in Snowflake.",
+  "permissions": [],
+  "backend": {
+    "entryPoints": [
+      "snowflake = snowflake_semantic_layer:SnowflakeSemanticLayer"
+    ],
+    "files": [
+      "backend/src/**/*.py"
+    ]
+  }
+}

superset/extensions/utils.py

@@ -185,6 +185,73 @@ def build_extension_data(extension: LoadedExtension) -> dict[str, Any]:
     return extension_data
 
 
+def load_extension_backend(extension: LoadedExtension) -> None:
+    """
+    Load an extension's backend code and register its entry points.
+
+    This installs the extension's Python modules in-memory and registers
+    any entry points declared in the manifest (e.g., semantic layers).
+    """
+    # Install backend modules in-memory if present
+    if extension.backend:
+        install_in_memory_importer(extension.backend)
+
+    # Register entry points from manifest
+    manifest = extension.manifest
+    if backend := manifest.get("backend"):
+        for ep_str in backend.get("entryPoints", []):
+            _register_entry_point(ep_str, extension.name)
+
+
+def _register_entry_point(ep_str: str, extension_name: str) -> None:
+    """
+    Parse and register a single entry point string.
+
+    Entry point format: "name = module:ClassName"
+    """
+    if "=" not in ep_str:
+        logger.warning(
+            "Invalid entry point format in extension %s: %s",
+            extension_name,
+            ep_str,
+        )
+        return
+
+    name, _, target = ep_str.partition("=")
+    name = name.strip()
+    target = target.strip()
+
+    if ":" not in target:
+        logger.warning(
+            "Invalid entry point target in extension %s: %s (expected module:Class)",
+            extension_name,
+            target,
+        )
+        return
+
+    module_path, _, class_name = target.partition(":")
+
+    try:
+        module = eager_import(module_path)
+        cls = getattr(module, class_name)
+
+        # Register with semantic layer registry
+        from superset.semantic_layers.registry import register_semantic_layer
+
+        register_semantic_layer(name, cls)
+        logger.info(
+            "Registered entry point '%s' from extension %s",
+            name,
+            extension_name,
+        )
+    except Exception:
+        logger.exception(
+            "Failed to load entry point '%s' from extension %s",
+            name,
+            extension_name,
+        )
+
+
 def get_extensions() -> dict[str, LoadedExtension]:
     extensions: dict[str, LoadedExtension] = {}
 
@@ -194,6 +261,7 @@ def get_extensions() -> dict[str, LoadedExtension]:
         extension = get_loaded_extension(files)
         extension_id = extension.manifest["id"]
         extensions[extension_id] = extension
+        load_extension_backend(extension)
         logger.info(
             "Loading extension %s (ID: %s) from local filesystem",
             extension.name,
@@ -208,6 +276,7 @@ def get_extensions() -> dict[str, LoadedExtension]:
             extension_id = extension.manifest["id"]
             if extension_id not in extensions:  # Don't override LOCAL_EXTENSIONS
                 extensions[extension_id] = extension
+                load_extension_backend(extension)
                 logger.info(
                     "Loading extension %s (ID: %s) from discovery path",
                     extension.name,

superset/initialization/__init__.py

@@ -527,33 +527,17 @@ class SupersetAppInitializer:  # pylint: disable=too-many-public-methods
         initialize_core_api_dependencies()
 
     def init_extensions(self) -> None:
-        from superset.extensions.utils import (
-            eager_import,
-            get_extensions,
-            install_in_memory_importer,
-        )
+        from superset.extensions.utils import get_extensions
 
         try:
-            extensions = get_extensions()
+            # get_extensions() handles loading backend modules and registering
+            # entry points via load_extension_backend()
+            get_extensions()
         except Exception:  # pylint: disable=broad-except  # noqa: S110
             # If the db hasn't been initialized yet, an exception will be raised.
             # It's fine to ignore this, as in this case there are no extensions
             # present yet.
-            return
-
-        for extension in extensions.values():
-            if backend_files := extension.backend:
-                install_in_memory_importer(backend_files)
-
-            backend = extension.manifest.get("backend")
-
-            if backend and (entrypoints := backend.get("entryPoints")):
-                for entrypoint in entrypoints:
-                    try:
-                        eager_import(entrypoint)
-                    except Exception as ex:  # pylint: disable=broad-except  # noqa: S110
-                        # Surface exceptions during initialization of extensions
-                        print(ex)
+            pass
 
     def init_app_in_ctx(self) -> None:
         """

superset/semantic_layers/models.py

@@ -23,7 +23,6 @@ import uuid
 from collections.abc import Hashable
 from dataclasses import dataclass
 from functools import cached_property
-from importlib.metadata import entry_points
 from typing import Any, TYPE_CHECKING
 
 from flask_appbuilder import Model
@@ -37,6 +36,7 @@ from superset.explorables.base import TimeGrainDict
 from superset.extensions import encrypted_field_factory
 from superset.models.helpers import AuditMixinNullable, QueryResult
 from superset.semantic_layers.mapper import get_results
+from superset.semantic_layers.registry import get_semantic_layer
 from superset.semantic_layers.types import (
     BINARY,
     BOOLEAN,
@@ -141,19 +141,11 @@ class SemanticLayer(AuditMixinNullable, Model):
         """
         Return semantic layer implementation.
         """
-        entry_point = next(
-            iter(
-                entry_points(
-                    group="superset.semantic_layers",
-                    name=self.type,
-                )
-            )
-        )
-        implementation_class = entry_point.load()
+        implementation_class = get_semantic_layer(self.type)
 
         if not issubclass(implementation_class, SemanticLayerImplementation):
             raise TypeError(
-                f"Entry point for semantic layer type '{self.type}' "
+                f"Semantic layer type '{self.type}' "
                 "must be a subclass of SemanticLayerImplementation"
             )
 

superset/semantic_layers/registry.py

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Semantic layer registry.
+
+This module provides a registry for semantic layer implementations that can be
+populated from:
+1. Standard Python entry points (for pip-installed packages)
+2. Superset extensions (for .supx bundles)
+"""
+
+from __future__ import annotations
+
+import logging
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superset.semantic_layers.types import SemanticLayerImplementation
+
+logger = logging.getLogger(__name__)
+
+ENTRY_POINT_GROUP = "superset.semantic_layers"
+
+# Registry mapping semantic layer type names to implementation classes
+_semantic_layer_registry: dict[str, type[SemanticLayerImplementation]] = {}
+_initialized_from_entry_points = False
+
+
+def _init_from_entry_points() -> None:
+    """
+    Pre-populate the registry from installed packages' entry points.
+
+    This is called lazily on first access to ensure all packages are loaded.
+    """
+    global _initialized_from_entry_points
+    if _initialized_from_entry_points:
+        return
+
+    for ep in entry_points(group=ENTRY_POINT_GROUP):
+        if ep.name not in _semantic_layer_registry:
+            try:
+                _semantic_layer_registry[ep.name] = ep.load()
+                logger.info(
+                    "Registered semantic layer '%s' from entry point %s",
+                    ep.name,
+                    ep.value,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to load semantic layer '%s' from entry point %s",
+                    ep.name,
+                    ep.value,
+                )
+
+    _initialized_from_entry_points = True
+
+
+def register_semantic_layer(
+    name: str,
+    cls: type[SemanticLayerImplementation],
+) -> None:
+    """
+    Register a semantic layer implementation.
+
+    This is called by extensions to register their semantic layer implementations.
+
+    Args:
+        name: The type name for the semantic layer (e.g., "snowflake")
+        cls: The implementation class
+    """
+    if name in _semantic_layer_registry:
+        logger.warning(
+            "Semantic layer '%s' already registered, overwriting with %s",
+            name,
+            cls,
+        )
+    _semantic_layer_registry[name] = cls
+    logger.info("Registered semantic layer '%s' from extension: %s", name, cls)
+
+
+def get_semantic_layer(name: str) -> type[SemanticLayerImplementation]:
+    """
+    Get a semantic layer implementation by name.
+
+    Args:
+        name: The type name for the semantic layer (e.g., "snowflake")
+
+    Returns:
+        The implementation class
+
+    Raises:
+        KeyError: If no implementation is registered for the given name
+    """
+    _init_from_entry_points()
+
+    if name not in _semantic_layer_registry:
+        available = ", ".join(_semantic_layer_registry.keys()) or "(none)"
+        raise KeyError(
+            f"No semantic layer implementation registered for type '{name}'. "
+            f"Available types: {available}"
+        )
+
+    return _semantic_layer_registry[name]
+
+
+def get_registered_semantic_layers() -> dict[str, type[SemanticLayerImplementation]]:
+    """
+    Get all registered semantic layer implementations.
+
+    Returns:
+        A dictionary mapping type names to implementation classes
+    """
+    _init_from_entry_points()
+    return dict(_semantic_layer_registry)