fix(native-filters): add support for versioned import/export (#16017)

* fix(native-filters): add support for versioned import/export

* fix lint

* address CR

* make query_context nullable

superset/charts/commands/export.py

@@ -54,7 +54,7 @@ class ExportChartsCommand(ExportModelsCommand):
             export_uuids=True,
         )
         # TODO (betodealmeida): move this logic to export_to_dict once this
-        # becomes the default export endpoint
+        #  becomes the default export endpoint
         for key in REMOVE_KEYS:
             del payload[key]
         if payload.get("params"):

superset/charts/schemas.py

@@ -1205,7 +1205,7 @@ class ImportV1ChartSchema(Schema):
     slice_name = fields.String(required=True)
     viz_type = fields.String(required=True)
     params = fields.Dict()
-    query_context = fields.Dict()
+    query_context = fields.String(allow_none=True, validate=utils.validate_json)
     cache_timeout = fields.Integer(allow_none=True)
     uuid = fields.UUID(required=True)
     version = fields.String(required=True)

superset/commands/importers/v1/examples.py

@@ -130,7 +130,7 @@ class ImportExamplesCommand(ImportModelsCommand):
         dashboard_chart_ids: List[Tuple[int, int]] = []
         for file_name, config in configs.items():
             if file_name.startswith("dashboards/"):
-                config = update_id_refs(config, chart_ids)
+                config = update_id_refs(config, chart_ids, dataset_info)
                 dashboard = import_dashboard(session, config, overwrite=overwrite)
                 dashboard.published = True
 

superset/dashboards/commands/export.py

@@ -30,6 +30,8 @@ from superset.dashboards.commands.exceptions import DashboardNotFoundError
 from superset.dashboards.commands.importers.v1.utils import find_chart_uuids
 from superset.dashboards.dao import DashboardDAO
 from superset.commands.export import ExportModelsCommand
+from superset.datasets.commands.export import ExportDatasetsCommand
+from superset.datasets.dao import DatasetDAO
 from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
 from superset.utils.dict_import_export import EXPORT_VERSION
@@ -116,7 +118,7 @@ class ExportDashboardsCommand(ExportModelsCommand):
             export_uuids=True,
         )
         # TODO (betodealmeida): move this logic to export_to_dict once this
-        # becomes the default export endpoint
+        #  becomes the default export endpoint
         for key, new_name in JSON_KEYS.items():
             value: Optional[str] = payload.pop(key, None)
             if value:
@@ -126,6 +128,18 @@ class ExportDashboardsCommand(ExportModelsCommand):
                     logger.info("Unable to decode `%s` field: %s", key, value)
                     payload[new_name] = {}
 
+        # Extract all native filter datasets and replace native
+        # filter dataset references with uuid
+        for native_filter in payload.get("metadata", {}).get(
+            "native_filter_configuration", []
+        ):
+            for target in native_filter.get("targets", []):
+                dataset_id = target.pop("datasetId", None)
+                if dataset_id is not None:
+                    dataset = DatasetDAO.find_by_id(dataset_id)
+                    target["datasetUuid"] = str(dataset.uuid)
+                    yield from ExportDatasetsCommand([dataset_id]).run()
+
         # the mapping between dashboard -> charts is inferred from the position
         # attribute, so if it's not present we need to add a default config
         if not payload.get("position"):

superset/dashboards/commands/importers/v1/__init__.py

@@ -27,6 +27,7 @@ from superset.commands.importers.v1 import ImportModelsCommand
 from superset.dashboards.commands.exceptions import DashboardImportError
 from superset.dashboards.commands.importers.v1.utils import (
     find_chart_uuids,
+    find_native_filter_datasets,
     import_dashboard,
     update_id_refs,
 )
@@ -60,14 +61,15 @@ class ImportDashboardsCommand(ImportModelsCommand):
     def _import(
         session: Session, configs: Dict[str, Any], overwrite: bool = False
     ) -> None:
-        # discover charts associated with dashboards
+        # discover charts and datasets associated with dashboards
         chart_uuids: Set[str] = set()
+        dataset_uuids: Set[str] = set()
         for file_name, config in configs.items():
             if file_name.startswith("dashboards/"):
                 chart_uuids.update(find_chart_uuids(config["position"]))
+                dataset_uuids.update(find_native_filter_datasets(config["metadata"]))
 
         # discover datasets associated with charts
-        dataset_uuids: Set[str] = set()
         for file_name, config in configs.items():
             if file_name.startswith("charts/") and config["uuid"] in chart_uuids:
                 dataset_uuids.add(config["dataset_uuid"])
@@ -96,7 +98,7 @@ class ImportDashboardsCommand(ImportModelsCommand):
                 dataset = import_dataset(session, config, overwrite=False)
                 dataset_info[str(dataset.uuid)] = {
                     "datasource_id": dataset.id,
-                    "datasource_type": "view" if dataset.is_sqllab_view else "table",
+                    "datasource_type": dataset.datasource_type,
                     "datasource_name": dataset.table_name,
                 }
 
@@ -121,7 +123,7 @@ class ImportDashboardsCommand(ImportModelsCommand):
         dashboard_chart_ids: List[Tuple[int, int]] = []
         for file_name, config in configs.items():
             if file_name.startswith("dashboards/"):
-                config = update_id_refs(config, chart_ids)
+                config = update_id_refs(config, chart_ids, dataset_info)
                 dashboard = import_dashboard(session, config, overwrite=overwrite)
                 for uuid in find_chart_uuids(config["position"]):
                     if uuid not in chart_ids:

superset/dashboards/commands/importers/v1/utils.py

@@ -33,6 +33,17 @@ def find_chart_uuids(position: Dict[str, Any]) -> Set[str]:
     return set(build_uuid_to_id_map(position))
 
 
+def find_native_filter_datasets(metadata: Dict[str, Any]) -> Set[str]:
+    uuids: Set[str] = set()
+    for native_filter in metadata.get("native_filter_configuration", []):
+        targets = native_filter.get("targets", [])
+        for target in targets:
+            dataset_uuid = target.get("datasetUuid")
+            if dataset_uuid:
+                uuids.add(dataset_uuid)
+    return uuids
+
+
 def build_uuid_to_id_map(position: Dict[str, Any]) -> Dict[str, int]:
     return {
         child["meta"]["uuid"]: child["meta"]["chartId"]
@@ -45,7 +56,11 @@ def build_uuid_to_id_map(position: Dict[str, Any]) -> Dict[str, int]:
     }
 
 
-def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[str, Any]:
+def update_id_refs(
+    config: Dict[str, Any],
+    chart_ids: Dict[str, int],
+    dataset_info: Dict[str, Dict[str, Any]],
+) -> Dict[str, Any]:
     """Update dashboard metadata to use new IDs"""
     fixed = config.copy()
 
@@ -103,6 +118,17 @@ def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[st
         ):
             child["meta"]["chartId"] = chart_ids[child["meta"]["uuid"]]
 
+    # fix native filter references
+    native_filter_configuration = fixed["metadata"].get(
+        "native_filter_configuration", []
+    )
+    for native_filter in native_filter_configuration:
+        targets = native_filter.get("targets", [])
+        for target in targets:
+            dataset_uuid = target.pop("datasetUuid", None)
+            if dataset_uuid:
+                target["datasetId"] = dataset_info[dataset_uuid]["datasource_id"]
+
     return fixed