From 02dbaeb9cf19a778ea5a9bd61c09999b37f61b7b Mon Sep 17 00:00:00 2001 From: Amin Ghadersohi Date: Mon, 6 Apr 2026 11:26:16 -0700 Subject: [PATCH] polish(tier3-storage): docs, test coverage, API cleanup - Rename migration file to match the table it creates (extension_storage) - Replace unsafe assert patterns in API with proper None guards - Add direct import of ExtensionStorage into api.py (removes runtime import) - Add missing requestBody OpenAPI spec to set_user and set_resource - Add _key_to_fernet docstring - Add test_list_resource_filtered_by_category (parity with list_global/list_user) - Add docs/developer_docs/extensions/extension-points/persistent-storage.md covering all three scopes, REST API, frontend/backend usage examples, encryption, key rotation, and data model table Co-Authored-By: Claude Sonnet 4.6 --- .../extension-points/persistent-storage.md | 235 ++++++++++++++++++ superset/extension_storage/api.py | 61 +++-- superset/extension_storage/daos.py | 5 + ...f6a7b8c9d0_add_extension_storage_table.py} | 0 .../daos/test_extension_storage_dao.py | 22 ++ 5 files changed, 303 insertions(+), 20 deletions(-) create mode 100644 docs/developer_docs/extensions/extension-points/persistent-storage.md rename superset/migrations/versions/{2026-03-20_12-00_e5f6a7b8c9d0_add_extension_assets_table.py => 2026-03-20_12-00_e5f6a7b8c9d0_add_extension_storage_table.py} (100%) diff --git a/docs/developer_docs/extensions/extension-points/persistent-storage.md b/docs/developer_docs/extensions/extension-points/persistent-storage.md new file mode 100644 index 00000000000..4ce58ad3732 --- /dev/null +++ b/docs/developer_docs/extensions/extension-points/persistent-storage.md @@ -0,0 +1,235 @@ +--- +title: Persistent Storage +sidebar_position: 3 +--- + + + +# Persistent Storage (Tier 3) + +Extensions can store arbitrary key-value data in Superset's database without +contributing model code to the core codebase. The storage layer supports three +**scopes**, optional **category** filtering for cheap listing, optional +**at-rest encryption**, and **key rotation** with zero downtime. + +## Storage Scopes + +| Scope | When to use | `user_fk` | `resource_type` | +|---|---|---|---| +| **Global** | Configuration shared across all users of the extension | `null` | `null` | +| **User** | Per-user preferences or saved artefacts | set | `null` | +| **Resource** | State tied to a specific Superset resource (dashboard, chart, …) | any | set | + +## REST API + +All endpoints live under: + +``` +/api/v1/extensions/{publisher}/{name}/storage/persistent/ +``` + +### Global scope + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `.../global/` | List entries (`?page=0&page_size=25&category=…`) | +| `GET` | `.../global/{key}` | Read a value (response body is the raw bytes) | +| `PUT` | `.../global/{key}` | Create or update | +| `DELETE` | `.../global/{key}` | Delete | + +### User scope + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `.../user/` | List the authenticated user's entries | +| `GET` | `.../user/{key}` | Read a value | +| `PUT` | `.../user/{key}` | Create or update | +| `DELETE` | `.../user/{key}` | Delete | + +### Resource scope + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `.../resources/{type}/{uuid}/` | List entries for the resource | +| `GET` | `.../resources/{type}/{uuid}/{key}` | Read a value | +| `PUT` | `.../resources/{type}/{uuid}/{key}` | Create or update | +| `DELETE` | `.../resources/{type}/{uuid}/{key}` | Delete | + +## PUT request body + +All write endpoints accept a JSON body: + +```json +{ + "value": "", + "value_type": "application/json", + "category": "my-category", + "description": "Human-readable label for cheap listing", + "is_encrypted": false +} +``` + +| Field | Required | Default | Notes | +|---|---|---|---| +| `value` | yes | `""` | The payload to store | +| `value_type` | no | `application/json` | MIME type returned on GET | +| `category` | no | `null` | Used to filter list results | +| `description` | no | `null` | Returned in list results without fetching `value` | +| `is_encrypted` | no | `false` | Encrypts `value` at rest with Fernet | + +## List response + +```json +{ + "count": 42, + "result": [ + { + "key": "my-key", + "uuid": "...", + "value_type": "application/json", + "category": "my-category", + "description": "...", + "is_encrypted": false + } + ] +} +``` + +The `value` blob is **not** included in list responses. Fetch individual keys +to retrieve values. + +## Frontend usage + +From a TypeScript extension frontend, use standard `fetch` with the Superset +CSRF token: + +```typescript +const BASE = '/api/v1/extensions/acme/my-ext/storage/persistent/user'; + +// Save +await fetch(`${BASE}/my-key`, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': await authentication.getCSRFToken(), + }, + body: JSON.stringify({ + value: JSON.stringify({ foo: 'bar' }), + value_type: 'application/json', + category: 'preferences', + description: 'My display name', // shown in list without fetching value + }), +}); + +// List +const { result } = await fetch(`${BASE}/?category=preferences`).then(r => r.json()); + +// Read +const raw = await fetch(`${BASE}/my-key`).then(r => r.text()); +const data = JSON.parse(raw); + +// Delete +await fetch(`${BASE}/my-key`, { method: 'DELETE', headers: { 'X-CSRFToken': ... } }); +``` + +## Backend usage (Python) + +Within extension backend code, use `ExtensionStorageDAO` directly: + +```python +from superset_core.common.daos import ExtensionStorageDAO + +EXT_ID = "acme.my-ext" + +# Write +ExtensionStorageDAO.set(EXT_ID, "config", b'{"theme":"dark"}', category="global-config") + +# Read +value: bytes | None = ExtensionStorageDAO.get_value(EXT_ID, "config") + +# User-scoped +ExtensionStorageDAO.set(EXT_ID, "prefs", b'{"lang":"fr"}', user_fk=user_id) +value = ExtensionStorageDAO.get_value(EXT_ID, "prefs", user_fk=user_id) + +# Resource-scoped +ExtensionStorageDAO.set( + EXT_ID, "state", payload, + resource_type="dashboard", resource_uuid=str(dashboard.uuid), +) +``` + +:::note +`ExtensionStorageDAO` methods call `db.session.flush()` but do **not** commit. +The caller (or the `@transaction()` decorator on the API endpoint) owns the +transaction. +::: + +## Encryption at rest + +Set `"is_encrypted": true` in the PUT body (frontend) or `is_encrypted=True` +in `ExtensionStorageDAO.set()` (backend). Values are encrypted with +[Fernet](https://cryptography.io/en/latest/fernet/) using a key derived from +`SECRET_KEY` (or the first entry of `EXTENSION_STORAGE_ENCRYPTION_KEYS` if +configured). Decryption is transparent on read. + +### Key rotation + +To rotate encryption keys without downtime: + +1. **Prepend** the new key to `EXTENSION_STORAGE_ENCRYPTION_KEYS` in your + Superset config and **restart**. Existing encrypted rows are still readable + (old key is tried as a fallback); new writes use the new key. + + ```python + # superset_config.py + EXTENSION_STORAGE_ENCRYPTION_KEYS = [ + "new-strong-secret", # used for new encryptions + "old-secret", # still tried for decryption + ] + ``` + +2. **Re-encrypt** all existing rows with the new key: + + ```bash + superset rotate-extension-storage-keys + ``` + +3. **Remove** the old key from the list and restart again. + +## Data model + +The `extension_storage` table schema: + +| Column | Type | Notes | +|---|---|---| +| `id` | integer PK | | +| `uuid` | UUID | Unique, auto-generated | +| `extension_id` | varchar(255) | `{publisher}.{name}` | +| `user_fk` | integer FK | `null` = global or resource scope | +| `resource_type` | varchar(64) | e.g. `dashboard`, `chart` | +| `resource_uuid` | varchar(36) | UUID of the linked resource | +| `key` | varchar(255) | Identifier within a scope | +| `category` | varchar(64) | Optional grouping tag | +| `description` | text | Summary for list endpoints | +| `value` | binary (up to 16 MB) | The stored payload | +| `value_type` | varchar(255) | MIME type, default `application/json` | +| `is_encrypted` | boolean | Whether `value` is Fernet-encrypted | +| `created_on`, `changed_on` | datetime | Audit fields | +| `created_by_fk`, `changed_by_fk` | integer FK | Audit fields | diff --git a/superset/extension_storage/api.py b/superset/extension_storage/api.py index e45653fe3b5..19673797317 100644 --- a/superset/extension_storage/api.py +++ b/superset/extension_storage/api.py @@ -46,6 +46,7 @@ from flask import request, Response from flask_appbuilder.api import expose, protect, safe from superset.extension_storage.daos import ExtensionStorageDAO +from superset.extension_storage.models import ExtensionStorage from superset.superset_typing import FlaskResponse from superset.utils.core import get_user_id from superset.utils.decorators import transaction @@ -78,10 +79,7 @@ def _extension_id(publisher: str, name: str) -> str: return f"{publisher}.{name}" -def _entry_to_dict(entry: object) -> dict[str, object]: - from superset.extension_storage.models import ExtensionStorage - - assert isinstance(entry, ExtensionStorage) +def _entry_to_dict(entry: "ExtensionStorage") -> dict[str, object]: return { "key": entry.key, "value_type": entry.value_type, @@ -181,13 +179,11 @@ class ExtensionStorageRestApi(BaseSupersetApi): description: Not found """ ext_id = _extension_id(publisher, name) - value = ExtensionStorageDAO.get_value(ext_id, key) - if value is None: - return self.response(404, message="Not found") entry = ExtensionStorageDAO.get(ext_id, key) - assert entry is not None - mime = entry.value_type - return Response(value, status=200, mimetype=mime) + if entry is None: + return self.response(404, message="Not found") + value = ExtensionStorageDAO.get_value(ext_id, key) or b"" + return Response(value, status=200, mimetype=entry.value_type) @expose( "///storage/persistent/global/", @@ -366,11 +362,10 @@ class ExtensionStorageRestApi(BaseSupersetApi): if not user_id: return self.response(401, message="Authentication required") ext_id = _extension_id(publisher, name) - value = ExtensionStorageDAO.get_value(ext_id, key, user_fk=user_id) - if value is None: - return self.response(404, message="Not found") entry = ExtensionStorageDAO.get(ext_id, key, user_fk=user_id) - assert entry is not None + if entry is None: + return self.response(404, message="Not found") + value = ExtensionStorageDAO.get_value(ext_id, key, user_fk=user_id) or b"" return Response(value, status=200, mimetype=entry.value_type) @expose( @@ -398,9 +393,22 @@ class ExtensionStorageRestApi(BaseSupersetApi): in: path required: true schema: {type: string} + requestBody: + content: + application/json: + schema: + type: object + properties: + value: {type: string} + value_type: {type: string} + category: {type: string} + description: {type: string} + is_encrypted: {type: boolean} responses: 200: description: Success + 401: + description: Authentication required """ user_id = get_user_id() if not user_id: @@ -571,15 +579,17 @@ class ExtensionStorageRestApi(BaseSupersetApi): description: Not found """ ext_id = _extension_id(publisher, name) - value = ExtensionStorageDAO.get_value( - ext_id, key, resource_type=resource_type, resource_uuid=resource_uuid - ) - if value is None: - return self.response(404, message="Not found") entry = ExtensionStorageDAO.get( ext_id, key, resource_type=resource_type, resource_uuid=resource_uuid ) - assert entry is not None + if entry is None: + return self.response(404, message="Not found") + value = ( + ExtensionStorageDAO.get_value( + ext_id, key, resource_type=resource_type, resource_uuid=resource_uuid + ) + or b"" + ) return Response(value, status=200, mimetype=entry.value_type) @expose( @@ -622,6 +632,17 @@ class ExtensionStorageRestApi(BaseSupersetApi): in: path required: true schema: {type: string} + requestBody: + content: + application/json: + schema: + type: object + properties: + value: {type: string} + value_type: {type: string} + category: {type: string} + description: {type: string} + is_encrypted: {type: boolean} responses: 200: description: Success diff --git a/superset/extension_storage/daos.py b/superset/extension_storage/daos.py index 7079590a45f..70eba2f0a6d 100644 --- a/superset/extension_storage/daos.py +++ b/superset/extension_storage/daos.py @@ -32,6 +32,11 @@ logger = logging.getLogger(__name__) def _key_to_fernet(raw_key: str | bytes) -> Fernet: + """Derive a Fernet instance from an arbitrary-length secret string. + + SHA-256 compresses the key to exactly 32 bytes, which are then + base64url-encoded to satisfy Fernet's key format requirement. + """ if isinstance(raw_key, str): raw_key = raw_key.encode() return Fernet(base64.urlsafe_b64encode(hashlib.sha256(raw_key).digest())) diff --git a/superset/migrations/versions/2026-03-20_12-00_e5f6a7b8c9d0_add_extension_assets_table.py b/superset/migrations/versions/2026-03-20_12-00_e5f6a7b8c9d0_add_extension_storage_table.py similarity index 100% rename from superset/migrations/versions/2026-03-20_12-00_e5f6a7b8c9d0_add_extension_assets_table.py rename to superset/migrations/versions/2026-03-20_12-00_e5f6a7b8c9d0_add_extension_storage_table.py diff --git a/tests/unit_tests/daos/test_extension_storage_dao.py b/tests/unit_tests/daos/test_extension_storage_dao.py index 4f678e5a864..e5c3dc60779 100644 --- a/tests/unit_tests/daos/test_extension_storage_dao.py +++ b/tests/unit_tests/daos/test_extension_storage_dao.py @@ -211,6 +211,28 @@ def test_list_resource(session: Session) -> None: assert keys == {"ra", "rb"} +def test_list_resource_filtered_by_category(session: Session) -> None: + _make_entry( + session, + key="r1", + resource_type=RES_TYPE, + resource_uuid=RES_UUID, + category="state", + ) + _make_entry( + session, + key="r2", + resource_type=RES_TYPE, + resource_uuid=RES_UUID, + category="config", + ) + results = ExtensionStorageDAO.list_resource( + EXT_A, RES_TYPE, RES_UUID, category="state" + ) + assert len(results) == 1 + assert results[0].key == "r1" + + # ── Pagination ────────────────────────────────────────────────────────────────