chore: Extensions architecture POC (#31934)

Co-authored-by: Ville Brofeldt <ville.brofeldt@apple.com>
Co-authored-by: Ville Brofeldt <ville@Villes-MacBook-Pro-2024.local>
Co-authored-by: Ville Brofeldt <v_brofeldt@apple.com>

superset/extensions/utils.py

@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import importlib.abc
+import importlib.util
+import logging
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Any, Generator, Iterable, Tuple
+from zipfile import ZipFile
+
+from flask import current_app
+from superset_core.extensions.types import Manifest
+
+from superset.extensions.types import BundleFile, LoadedExtension
+from superset.utils import json
+from superset.utils.core import check_is_safe_zip
+
+logger = logging.getLogger(__name__)
+
+FRONTEND_REGEX = re.compile(r"^frontend/dist/([^/]+)$")
+BACKEND_REGEX = re.compile(r"^backend/src/(.+)$")
+
+
+class InMemoryLoader(importlib.abc.Loader):
+    def __init__(
+        self, module_name: str, source: str, is_package: bool, origin: str
+    ) -> None:
+        self.module_name = module_name
+        self.source = source
+        self.is_package = is_package
+        self.origin = origin
+
+    def exec_module(self, module: Any) -> None:
+        module.__file__ = self.origin
+        module.__package__ = (
+            self.module_name if self.is_package else self.module_name.rpartition(".")[0]
+        )
+        if self.is_package:
+            module.__path__ = []
+        exec(self.source, module.__dict__)  # noqa: S102
+
+
+class InMemoryFinder(importlib.abc.MetaPathFinder):
+    def __init__(self, file_dict: dict[str, bytes]) -> None:
+        self.modules: dict[str, Tuple[Any, Any, Any]] = {}
+        for path, content in file_dict.items():
+            mod_name, is_package = self._get_module_name(path)
+            self.modules[mod_name] = (content, is_package, path)
+
+    def _get_module_name(self, file_path: str) -> Tuple[str, bool]:
+        parts = list(Path(file_path).parts)
+        is_package = parts[-1] == "__init__.py"
+        if is_package:
+            parts = parts[:-1]
+        else:
+            parts[-1] = Path(parts[-1]).stem
+
+        mod_name = ".".join(parts)
+        return mod_name, is_package
+
+    def find_spec(self, fullname: str, path: Any, target: Any = None) -> Any | None:
+        if fullname in self.modules:
+            source, is_package, origin = self.modules[fullname]
+            return importlib.util.spec_from_loader(
+                fullname,
+                InMemoryLoader(fullname, source, is_package, origin),
+                origin=origin,
+                is_package=is_package,
+            )
+        return None
+
+
+def install_in_memory_importer(file_dict: dict[str, bytes]) -> None:
+    finder = InMemoryFinder(file_dict)
+    sys.meta_path.insert(0, finder)
+
+
+def eager_import(module_name: str) -> Any:
+    if module_name in sys.modules:
+        return sys.modules[module_name]
+    return importlib.import_module(module_name)
+
+
+def get_bundle_files_from_zip(zip_file: ZipFile) -> Generator[BundleFile, None, None]:
+    check_is_safe_zip(zip_file)
+    for name in zip_file.namelist():
+        content = zip_file.read(name)
+        yield BundleFile(name=name, content=content)
+
+
+def get_bundle_files_from_path(base_path: str) -> Generator[BundleFile, None, None]:
+    dist_path = os.path.join(base_path, "dist")
+
+    if not os.path.isdir(dist_path):
+        raise Exception(f"Expected directory {dist_path} does not exist.")
+
+    for root, _, files in os.walk(dist_path):
+        for file in files:
+            full_path = os.path.join(root, file)
+            rel_path = os.path.relpath(full_path, dist_path).replace(os.sep, "/")
+            with open(full_path, "rb") as f:
+                content = f.read()
+            yield BundleFile(name=rel_path, content=content)
+
+
+def get_loaded_extension(files: Iterable[BundleFile]) -> LoadedExtension:
+    manifest: Manifest = {}
+    frontend: dict[str, bytes] = {}
+    backend: dict[str, bytes] = {}
+
+    for file in files:
+        filename = file.name
+        content = file.content
+
+        if filename == "manifest.json":
+            try:
+                manifest = json.loads(content)
+                if "id" not in manifest:
+                    raise Exception("Missing 'id' in manifest")
+                if "name" not in manifest:
+                    raise Exception("Missing 'name' in manifest")
+            except Exception as e:
+                raise Exception(f"Invalid manifest.json: {e}") from e
+
+        elif (match := FRONTEND_REGEX.match(filename)) is not None:
+            frontend[match.group(1)] = content
+
+        elif (match := BACKEND_REGEX.match(filename)) is not None:
+            backend[match.group(1)] = content
+
+        else:
+            raise Exception(f"Unexpected file in bundle: {filename}")
+
+    id_ = manifest["id"]
+    name = manifest["name"]
+    version = manifest["version"]
+    return LoadedExtension(
+        id=id_,
+        name=name,
+        manifest=manifest,
+        frontend=frontend,
+        backend=backend,
+        version=version,
+    )
+
+
+def build_extension_data(extension: LoadedExtension) -> dict[str, Any]:
+    manifest: Manifest = extension.manifest
+    extension_data: dict[str, Any] = {
+        "id": manifest["id"],
+        "name": extension.name,
+        "version": extension.version,
+        "description": manifest.get("description", ""),
+        "dependencies": manifest.get("dependencies", []),
+        "extensionDependencies": manifest.get("extensionDependencies", []),
+    }
+    if frontend := manifest.get("frontend"):
+        module_federation = frontend.get("moduleFederation", {})
+        remote_entry = frontend["remoteEntry"]
+        extension_data.update(
+            {
+                "remoteEntry": f"/api/v1/extensions/{manifest['id']}/{remote_entry}",  # noqa: E501
+                "exposedModules": module_federation.get("exposes", []),
+                "contributions": frontend.get("contributions", {}),
+            }
+        )
+    return extension_data
+
+
+def get_extensions() -> dict[str, LoadedExtension]:
+    extensions: dict[str, LoadedExtension] = {}
+
+    # Load extensions from LOCAL_EXTENSIONS configuration (filesystem paths)
+    for path in current_app.config["LOCAL_EXTENSIONS"]:
+        files = get_bundle_files_from_path(path)
+        extension = get_loaded_extension(files)
+        extension_id = extension.manifest["id"]
+        extensions[extension_id] = extension
+        logger.info(
+            f"Loading extension {extension.name} (ID: {extension_id}) "
+            "from local filesystem"
+        )
+
+    # Load extensions from discovery path (.supx files)
+    if extensions_path := current_app.config.get("EXTENSIONS_PATH"):
+        from superset.extensions.discovery import discover_and_load_extensions
+
+        for extension in discover_and_load_extensions(extensions_path):
+            extension_id = extension.manifest["id"]
+            if extension_id not in extensions:  # Don't override LOCAL_EXTENSIONS
+                extensions[extension_id] = extension
+                logger.info(
+                    f"Loading extension {extension.name} (ID: {extension_id}) "
+                    "from discovery path"
+                )
+            else:
+                logger.info(
+                    f"Extension {extension.name} (ID: {extension_id}) already "
+                    "loaded from LOCAL_EXTENSIONS, skipping discovery version"
+                )
+
+    return extensions