diff --git a/UPDATING.md b/UPDATING.md index e12c06540fc..4e829bcf8de 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,31 @@ assists people when migrating to a new version. ## Next +### Dashboard "Export Data to Excel" requires a Celery worker and S3 bucket + +A new dashboard action exports every chart's data to a single multi-sheet +`.xlsx` asynchronously. It is disabled by default and turns on only when +`EXCEL_EXPORT_S3_BUCKET` is set (the endpoint returns `501` otherwise). It also +requires a running Celery worker and a configured SMTP transport, since the task +emails the requesting user a pre-signed download link. New config keys: +`EXCEL_EXPORT_S3_BUCKET`, `EXCEL_EXPORT_S3_KEY_PREFIX`, +`EXCEL_EXPORT_LINK_TTL_SECONDS`, `EXCEL_EXPORT_S3_CLIENT_KWARGS`, and +`EXCEL_EXPORT_TABLE_VIZ_TYPES`. + +The feature depends on `boto3`, which is **not** installed by default; install it +with `pip install apache-superset[excel-export]`. + +A second mode, **Export Images to Excel**, embeds non-table charts as rendered +images (which viz types stay tabular is controlled by +`EXCEL_EXPORT_TABLE_VIZ_TYPES`). It renders through the headless webdriver, so the +menu option only appears when the webdriver screenshot feature flags +(`ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS`, +`ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT`) are enabled. + +Deployments that override `CELERY_CONFIG` must add +`"superset.tasks.export_dashboard_excel"` to their `imports` tuple, or the task +will not register and exports will silently never run. + ### SQL_QUERY_MUTATOR now honors MUTATE_AFTER_SPLIT in SQL Lab SQL Lab now applies `SQL_QUERY_MUTATOR` according to `MUTATE_AFTER_SPLIT`, matching the documented semantics and the chart/query path. This only affects deployments that define `SQL_QUERY_MUTATOR` in `superset_config.py`: diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index ce7f1998708..7755cdb2cdc 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -87,6 +87,7 @@ class CeleryConfig: "superset.tasks.scheduler", "superset.tasks.thumbnails", "superset.tasks.cache", + "superset.tasks.export_dashboard_excel", ) result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" worker_prefetch_multiplier = 1 diff --git a/docs/docs/using-superset/exporting-dashboard-data.mdx b/docs/docs/using-superset/exporting-dashboard-data.mdx new file mode 100644 index 00000000000..46c0a749d7c --- /dev/null +++ b/docs/docs/using-superset/exporting-dashboard-data.mdx @@ -0,0 +1,99 @@ +--- +title: Exporting Dashboard Data to Excel +hide_title: true +sidebar_position: 7 +version: 1 +--- + +# Exporting Dashboard Data to Excel + +Superset can export every chart on a dashboard to a single Excel workbook, with +each chart's underlying data rendered as its own worksheet. The export reflects +the dashboard's currently applied filters and runs asynchronously: when it +finishes, the requesting user receives an email with a time-limited download +link. + +## Using the export + +From a dashboard, open the **... (actions) → Download** submenu and choose +**Export Data to Excel**. The action appears for users who have the dashboard +`can_export` permission. You'll see a confirmation that the export is being +prepared; the workbook arrives by email when it's ready. + +A second option, **Export Images to Excel**, embeds each non-table chart as a +rendered image (tables stay tabular) instead of exporting raw data. Because it +renders charts through the headless webdriver, this option only appears when the +webdriver screenshot feature flags are enabled (see the prerequisites below); +which viz types stay tabular is controlled by `EXCEL_EXPORT_TABLE_VIZ_TYPES`. + +Notes on the generated workbook: + +- One worksheet per chart, named `{chart_id} - {chart title}` (truncated to + Excel's 31-character limit; the chart id keeps names unique). +- Charts nested in tabs are included. +- Data reflects the dashboard's active filter state at the time of export. +- A chart with no saved query context is skipped and listed in the email; open + the chart in Explore and re-save it to include it next time. +- Row counts per sheet are capped the same way as the chart-level CSV/Excel + export (`ROW_LIMIT`, bounded by `SQL_MAX_ROW`), and never exceed Excel's + per-sheet maximum. + +## Prerequisites + +This feature is **disabled by default**. It requires: + +1. **The `boto3` dependency.** It is not installed by default; install it with + `pip install apache-superset[excel-export]`. Without it, exports fail and the + user receives a failure email. +2. **An S3 bucket.** Set `EXCEL_EXPORT_S3_BUCKET`. Until it is set, the export + endpoint returns `501` and the menu action surfaces a "not configured" + message. +3. **A running Celery worker.** The export runs as a Celery task. If no worker + is running, the request is accepted but nothing is produced. +4. **A configured SMTP transport.** The download link is delivered by email + using the same settings as alerts & reports (`SMTP_*`, + `EMAIL_REPORTS_SUBJECT_PREFIX`). + +**Export Images to Excel** additionally requires a working headless webdriver — +the same infrastructure scheduled reports and thumbnails use (`WEBDRIVER_*`, +plus the `ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS` and +`ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT` feature flags). The menu option +is hidden when those flags are off; if the webdriver is unreachable, image +charts come back empty even though the data export path still works. + +Deployments that override `CELERY_CONFIG` must add +`"superset.tasks.export_dashboard_excel"` to the `imports` tuple, or the task +will not register. + +## Configuration keys + +| Key | Default | Description | +| --- | --- | --- | +| `EXCEL_EXPORT_S3_BUCKET` | `None` | Destination bucket. Required; `501` if unset. | +| `EXCEL_EXPORT_S3_KEY_PREFIX` | `"dashboard-exports/"` | Key prefix: `{prefix}{dashboard_id}/{job_id}.xlsx`. | +| `EXCEL_EXPORT_LINK_TTL_SECONDS` | `86400` | Lifetime of the pre-signed download URL (24h). | +| `EXCEL_EXPORT_S3_CLIENT_KWARGS` | `{}` | Extra kwargs for `boto3.client("s3", ...)` — e.g. `region_name`, or `endpoint_url` for MinIO/LocalStack. | +| `EXCEL_EXPORT_TABLE_VIZ_TYPES` | `None` | Viz types kept tabular in **Export Images to Excel** mode; every other type is embedded as an image. `None` uses the built-in default (`table`, `pivot_table`, `pivot_table_v2`). | + +Credentials and region resolve through the standard boto3 chain (environment +variables, shared config, or instance role) unless overridden via +`EXCEL_EXPORT_S3_CLIENT_KWARGS`. The worker needs `s3:PutObject` on the bucket. + +## Security considerations + +- The emailed link is a **pre-signed S3 URL**: anyone who holds it can download + the workbook until it expires. Keep the bucket **private**, enable + encryption, and consider a lifecycle rule to delete objects after a few days. + Lower `EXCEL_EXPORT_LINK_TTL_SECONDS` if 24 hours is too long for your data. +- The export runs with the requesting user's permissions; each chart's query is + access-checked, so users only ever receive data they are entitled to. + +## Limitations + +- **Embedded dashboards / guest tokens are not supported** in this version, + because guest users have no email address to deliver the link to. Logged-in + users viewing an embedded dashboard can still use the export. +- The default **Export Data to Excel** mode exports data only (no visual + styling). Use **Export Images to Excel** to embed rendered chart images, which + requires the webdriver infrastructure described in the prerequisites. +- Scheduled/automated exports are not part of this feature. diff --git a/pyproject.toml b/pyproject.toml index 976b8d82ee3..1d3803c1fee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,6 +153,10 @@ solr = ["sqlalchemy-solr >= 0.2.4.3"] elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"] exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"] excel = ["xlrd>=2.0.2, <2.1"] +# Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and +# emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so +# installing this extra is only required to actually run exports. +excel-export = ["boto3"] fastmcp = [ "fastmcp>=3.4.3,<4.0", # tiktoken backs the response-size-guard token estimator. Without diff --git a/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/DownloadMenuItems.test.tsx b/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/DownloadMenuItems.test.tsx index 86efc2860e0..c06cd6a06df 100644 --- a/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/DownloadMenuItems.test.tsx +++ b/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/DownloadMenuItems.test.tsx @@ -26,6 +26,7 @@ import { import { Menu, MenuItem } from '@superset-ui/core/components/Menu'; import { FeatureFlag, + getClientErrorObject, isFeatureEnabled, SupersetClient, } from '@superset-ui/core'; @@ -46,12 +47,15 @@ jest.mock('src/components/MessageToasts/withToasts', () => ({ jest.mock('@superset-ui/core', () => ({ ...jest.requireActual('@superset-ui/core'), isFeatureEnabled: jest.fn().mockReturnValue(false), + getClientErrorObject: jest.fn().mockResolvedValue({}), SupersetClient: { get: jest.fn(), + post: jest.fn(), }, })); const mockSupersetClient = SupersetClient as jest.Mocked; +const mockGetClientErrorObject = getClientErrorObject as jest.Mock; const createProps = () => ({ pdfMenuItemTitle: 'Export to PDF', @@ -70,19 +74,40 @@ const MenuWrapper = () => { return ; }; +const MenuWrapperWithProps = ( + overrides: Partial> & { + canExportImage?: boolean; + }, +) => { + const downloadMenuItem = useDownloadMenuItems({ + ...createProps(), + ...overrides, + }); + const menuItems: MenuItem[] = [downloadMenuItem]; + return ; +}; + const originalCreateObjectURL = window.URL.createObjectURL; const originalRevokeObjectURL = window.URL.revokeObjectURL; beforeEach(() => { jest.clearAllMocks(); + // Reset the implementation each test: clearAllMocks resets call history but + // not mockReturnValue, so an override in one test would otherwise leak. + (isFeatureEnabled as jest.Mock).mockReturnValue(false); }); +// "Export Images to Excel" is gated on the webdriver screenshot feature flags. +const enableWebDriverScreenshot = () => + (isFeatureEnabled as jest.Mock).mockReturnValue(true); + afterEach(() => { window.URL.createObjectURL = originalCreateObjectURL; window.URL.revokeObjectURL = originalRevokeObjectURL; }); test('Should render all menu items', () => { + enableWebDriverScreenshot(); render(, { useRedux: true, }); @@ -92,10 +117,120 @@ test('Should render all menu items', () => { expect(screen.getByText('Download as Image')).toBeInTheDocument(); // Export options + expect(screen.getByText('Export Data to Excel')).toBeInTheDocument(); + expect(screen.getByText('Export Images to Excel')).toBeInTheDocument(); expect(screen.getByText('Export YAML')).toBeInTheDocument(); expect(screen.getByText('Export as Example')).toBeInTheDocument(); }); +test('Export Images to Excel is hidden when the webdriver is not enabled', () => { + // Default: webdriver screenshot flags off. Image export needs the webdriver, + // so only the data export is offered. + render(, { useRedux: true }); + + expect(screen.getByText('Export Data to Excel')).toBeInTheDocument(); + expect(screen.queryByText('Export Images to Excel')).not.toBeInTheDocument(); +}); + +test('Excel export items are hidden when userCanExport is false', () => { + render(, { useRedux: true }); + + expect(screen.queryByText('Export Data to Excel')).not.toBeInTheDocument(); + expect(screen.queryByText('Export Images to Excel')).not.toBeInTheDocument(); + // YAML export is not gated and remains visible + expect(screen.getByText('Export YAML')).toBeInTheDocument(); +}); + +test('Export Data to Excel posts mode "data" and shows a pending toast', async () => { + mockSupersetClient.post.mockResolvedValue({ + json: { job_id: 'abc' }, + } as never); + + render(, { useRedux: true }); + + await userEvent.click(screen.getByText('Export Data to Excel')); + + await waitFor(() => { + expect(mockSupersetClient.post).toHaveBeenCalledWith({ + endpoint: '/api/v1/dashboard/123/export_xlsx/', + jsonPayload: { active_data_mask: {}, mode: 'data' }, + }); + expect(mockAddSuccessToast).toHaveBeenCalledWith( + "Your export is being prepared. You'll receive an email when it's ready.", + ); + }); +}); + +test('Export Images to Excel posts mode "images" and shows a pending toast', async () => { + enableWebDriverScreenshot(); + mockSupersetClient.post.mockResolvedValue({ + json: { job_id: 'abc' }, + } as never); + + render(, { useRedux: true }); + + await userEvent.click(screen.getByText('Export Images to Excel')); + + await waitFor(() => { + expect(mockSupersetClient.post).toHaveBeenCalledWith({ + endpoint: '/api/v1/dashboard/123/export_xlsx/', + jsonPayload: { active_data_mask: {}, mode: 'images' }, + }); + expect(mockAddSuccessToast).toHaveBeenCalledWith( + "Your export is being prepared. You'll receive an email when it's ready.", + ); + }); +}); + +test('Export Data to Excel shows an "already in progress" toast when throttled', async () => { + // The throttle response is 202 with a message but no job_id. + mockSupersetClient.post.mockResolvedValue({ + json: { + message: 'An Excel export for this dashboard is already in progress.', + }, + } as never); + + render(, { useRedux: true }); + + await userEvent.click(screen.getByText('Export Data to Excel')); + + await waitFor(() => { + expect(mockAddSuccessToast).toHaveBeenCalledWith( + 'An export for this dashboard is already in progress.', + ); + }); +}); + +test('Export Data to Excel shows a config error toast on 501', async () => { + mockSupersetClient.post.mockRejectedValue(new Error('not configured')); + mockGetClientErrorObject.mockResolvedValue({ status: 501 }); + + render(, { useRedux: true }); + + await userEvent.click(screen.getByText('Export Data to Excel')); + + await waitFor(() => { + expect(mockAddDangerToast).toHaveBeenCalledWith( + 'Excel export is not configured on this server.', + ); + }); +}); + +test('Export Data to Excel shows a generic error toast on other failures', async () => { + mockSupersetClient.post.mockRejectedValue(new Error('boom')); + mockGetClientErrorObject.mockResolvedValue({ status: 500 }); + + render(, { useRedux: true }); + + await userEvent.click(screen.getByText('Export Data to Excel')); + + await waitFor(() => { + expect(mockAddDangerToast).toHaveBeenCalledWith( + 'Sorry, something went wrong. Try again later.', + ); + }); +}); + test('Export as Example calls SupersetClient.get with correct endpoint', async () => { const mockBlob = new Blob(['test'], { type: 'application/zip' }); const mockResponse: Pick = { @@ -144,19 +279,6 @@ test('Export as Example shows error toast on failure', async () => { const mockIsFeatureEnabled = isFeatureEnabled as jest.Mock; -const MenuWrapperWithProps = ( - overrides: Partial> & { - canExportImage?: boolean; - }, -) => { - const downloadMenuItem = useDownloadMenuItems({ - ...createProps(), - ...overrides, - }); - const menuItems: MenuItem[] = [downloadMenuItem]; - return ; -}; - test('Screenshot menu items should be disabled when GranularExportControls is ON and canExportImage is false', () => { mockIsFeatureEnabled.mockImplementation( (flag: string) => flag === FeatureFlag.GranularExportControls, diff --git a/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/index.tsx b/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/index.tsx index 0379588bf7a..248d2795260 100644 --- a/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/index.tsx +++ b/superset-frontend/src/dashboard/components/menu/DownloadMenuItems/index.tsx @@ -17,17 +17,20 @@ * under the License. */ import { SyntheticEvent } from 'react'; +import { useSelector } from 'react-redux'; import { logging } from '@apache-superset/core/utils'; import { t } from '@apache-superset/core/translation'; import { FeatureFlag, + getClientErrorObject, isFeatureEnabled, SupersetClient, } from '@superset-ui/core'; import { MenuItem } from '@superset-ui/core/components/Menu'; import { parse as parseContentDisposition } from 'content-disposition'; import { useDownloadScreenshot } from 'src/dashboard/hooks/useDownloadScreenshot'; -import { MenuKeys } from 'src/dashboard/types'; +import { NATIVE_FILTER_PREFIX } from 'src/dashboard/components/nativeFilters/FiltersConfigModal/utils'; +import { MenuKeys, RootState } from 'src/dashboard/types'; import downloadAsPdf from 'src/utils/downloadAsPdf'; import downloadAsImage from 'src/utils/downloadAsImage'; import handleResourceExport from 'src/utils/export'; @@ -68,8 +71,19 @@ export const useDownloadMenuItems = ( } = props; const { addDangerToast, addSuccessToast } = useToasts(); + const dataMask = useSelector((state: RootState) => state.dataMask); const SCREENSHOT_NODE_SELECTOR = '.dashboard'; + const buildActiveDataMask = (): Record => + Object.entries(dataMask || {}).reduce< + Record + >((acc, [id, mask]) => { + if (id.startsWith(NATIVE_FILTER_PREFIX)) { + acc[id] = { extraFormData: mask?.extraFormData ?? {} }; + } + return acc; + }, {}); + const isWebDriverScreenshotEnabled = isFeatureEnabled(FeatureFlag.EnableDashboardScreenshotEndpoints) && isFeatureEnabled(FeatureFlag.EnableDashboardDownloadWebDriverScreenshot); @@ -153,6 +167,39 @@ export const useDownloadMenuItems = ( } }; + const onExportXlsx = async (mode: 'data' | 'images') => { + try { + const { json } = await SupersetClient.post({ + endpoint: `/api/v1/dashboard/${dashboardId}/export_xlsx/`, + jsonPayload: { active_data_mask: buildActiveDataMask(), mode }, + }); + // The throttle response (an export is already running) returns 202 with a + // message but no job_id; only a freshly enqueued job carries a job_id. + if ((json as { job_id?: string })?.job_id) { + addSuccessToast( + t( + "Your export is being prepared. You'll receive an email when it's ready.", + ), + ); + } else { + addSuccessToast( + t('An export for this dashboard is already in progress.'), + ); + } + } catch (error) { + // status comes from the response (Partial), which + // the union type does not expose uniformly; read it via a narrow cast. + const { status } = (await getClientErrorObject(error)) as { + status?: number; + }; + if (status === 501) { + addDangerToast(t('Excel export is not configured on this server.')); + } else { + addDangerToast(t('Sorry, something went wrong. Try again later.')); + } + } + }; + const imageDisabled = canExportImage === false; const imageExportLabel = (text: string) => @@ -198,6 +245,28 @@ export const useDownloadMenuItems = ( ]; const exportMenuItems: MenuItem[] = [ + ...(userCanExport + ? [ + { + key: 'export-xlsx', + label: t('Export Data to Excel'), + onClick: () => onExportXlsx('data'), + }, + // Image export renders charts through the headless webdriver, so only + // offer it where that infrastructure is available (same signal as the + // PDF/PNG image downloads above); otherwise non-table charts would + // silently come back empty. + ...(isWebDriverScreenshotEnabled + ? [ + { + key: 'export-xlsx-images', + label: t('Export Images to Excel'), + onClick: () => onExportXlsx('images'), + }, + ] + : []), + ] + : []), { key: 'export-yaml', label: t('Export YAML'), diff --git a/superset/charts/data/dashboard_filter_context.py b/superset/charts/data/dashboard_filter_context.py index bd36bd07432..1eda538b6fc 100644 --- a/superset/charts/data/dashboard_filter_context.py +++ b/superset/charts/data/dashboard_filter_context.py @@ -199,6 +199,31 @@ def _extract_filter_extra_form_data( return None, DashboardFilterStatus.NOT_APPLIED +def _resolve_filter_extra_form_data( + filter_config: dict[str, Any], + active_data_mask: dict[str, Any] | None, +) -> tuple[dict[str, Any] | None, DashboardFilterStatus]: + """ + Resolve a filter's extra_form_data and status, preferring an active value + from ``active_data_mask`` over the filter's saved default. + + When ``active_data_mask`` provides an entry for this filter, its + ``extraFormData`` is authoritative: a non-empty value is APPLIED, while an + empty value means the user explicitly cleared the filter (NOT_APPLIED, with + no fallback to the saved default). When no active entry exists, fall back to + the saved-default behavior in ``_extract_filter_extra_form_data``. + + Returns (extra_form_data, status). + """ + flt_id = filter_config.get("id", "") + if active_data_mask is not None and flt_id in active_data_mask: + active_efd = (active_data_mask[flt_id] or {}).get("extraFormData") or {} + if active_efd: + return active_efd, DashboardFilterStatus.APPLIED + return None, DashboardFilterStatus.NOT_APPLIED + return _extract_filter_extra_form_data(filter_config) + + def _get_filter_target_column(filter_config: dict[str, Any]) -> str | None: """Extract the target column name from a native filter configuration.""" if targets := filter_config.get("targets", []): @@ -244,16 +269,26 @@ def _check_dashboard_access(dashboard: Dashboard) -> None: def get_dashboard_filter_context( dashboard_id: int, chart_id: int, + *, + active_data_mask: dict[str, Any] | None = None, ) -> DashboardFilterContext: """ Build a DashboardFilterContext for a chart on a dashboard. Loads the dashboard's native filter configuration, determines which - filters are in scope for the given chart, extracts default filter values, + filters are in scope for the given chart, resolves each filter's value, and returns the merged extra_form_data along with metadata about each filter. + When ``active_data_mask`` is provided (e.g. the live filter state from a + dashboard view), each in-scope filter present in the mask uses its active + ``extraFormData`` instead of the saved default; an empty active value means + the filter was cleared. Filters absent from the mask fall back to their + saved defaults, so omitting ``active_data_mask`` reproduces the dashboard's + initial-load behavior. + :param dashboard_id: The ID of the dashboard :param chart_id: The ID of the chart + :param active_data_mask: Optional live filter state keyed by native filter id :returns: DashboardFilterContext with merged extra_form_data and filter metadata :raises ValueError: if dashboard not found or chart not on dashboard :raises SupersetSecurityException: if the user cannot access the dashboard @@ -287,7 +322,7 @@ def get_dashboard_filter_context( flt_id = flt.get("id", "") flt_name = flt.get("name", "") target_column = _get_filter_target_column(flt) - extra_form_data, status = _extract_filter_extra_form_data(flt) + extra_form_data, status = _resolve_filter_extra_form_data(flt, active_data_mask) if extra_form_data and status == DashboardFilterStatus.APPLIED: context.extra_form_data = _merge_extra_form_data( diff --git a/superset/config.py b/superset/config.py index a745f4f234a..bfee0039947 100644 --- a/superset/config.py +++ b/superset/config.py @@ -1443,6 +1443,27 @@ CSV_STREAMING_ROW_THRESHOLD = 100000 # note: index option should not be overridden EXCEL_EXPORT: dict[str, Any] = {} +# --------------------------------------------------- +# Dashboard "Export Data to Excel" (async, S3-backed) +# --------------------------------------------------- +# Destination S3 bucket for generated dashboard .xlsx exports. The feature is +# disabled until this is set: the export endpoint returns 501 when it is None. +EXCEL_EXPORT_S3_BUCKET: str | None = None +# Key prefix for export objects: {prefix}{dashboard_id}/{job_id}.xlsx +EXCEL_EXPORT_S3_KEY_PREFIX = "dashboard-exports/" +# Lifetime (seconds) of the pre-signed download URL emailed to the user (24h). +# Note: AWS S3 caps pre-signed URL lifetime at 7 days (604800 seconds); larger +# values are rejected by S3, so keep this at or below that when using AWS. +EXCEL_EXPORT_LINK_TTL_SECONDS = 86400 +# Extra kwargs passed to boto3.client("s3", ...) — e.g. region_name, or an +# endpoint_url for S3-compatible stores (MinIO/LocalStack). Credentials +# otherwise resolve through the standard boto3 chain. +EXCEL_EXPORT_S3_CLIENT_KWARGS: dict[str, Any] = {} +# Viz types treated as tables in the "Export Images to Excel" mode: these charts +# stay tabular (one worksheet of data) while every other viz type is embedded as +# a rendered image. Set to None to fall back to the built-in default. +EXCEL_EXPORT_TABLE_VIZ_TYPES: set[str] | None = None + # --------------------------------------------------- # Time grain configurations # --------------------------------------------------- @@ -1635,6 +1656,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods "superset.tasks.thumbnails", "superset.tasks.cache", "superset.tasks.slack", + "superset.tasks.export_dashboard_excel", ) result_backend = "db+sqlite:///celery_results.sqlite" worker_prefetch_multiplier = 1 diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py index 45fb970a57b..5a138d0d401 100644 --- a/superset/dashboards/api.py +++ b/superset/dashboards/api.py @@ -17,6 +17,7 @@ # pylint: disable=too-many-lines import functools import logging +import uuid from datetime import datetime from io import BytesIO from typing import Any, Callable, cast @@ -82,6 +83,8 @@ from superset.commands.dashboard.update import ( UpdateDashboardNativeFiltersCommand, ) from superset.commands.database.exceptions import DatasetValidationError +from superset.commands.distributed_lock.acquire import AcquireDistributedLock +from superset.commands.distributed_lock.release import ReleaseDistributedLock from superset.commands.exceptions import TagForbiddenError from superset.commands.importers.exceptions import NoValidFilesFoundError from superset.commands.importers.v1.utils import get_contents_from_bundle @@ -107,6 +110,8 @@ from superset.dashboards.schemas import ( DashboardColorsConfigUpdateSchema, DashboardCopySchema, DashboardDatasetSchema, + DashboardExportXlsxPostSchema, + DashboardExportXlsxResponseSchema, DashboardGetResponseSchema, DashboardNativeFiltersConfigUpdateSchema, DashboardPostSchema, @@ -124,7 +129,9 @@ from superset.dashboards.schemas import ( thumbnail_query_schema, ) from superset.exceptions import ( + LockAlreadyHeldException, ScreenshotImageNotAvailableException, + SupersetSecurityException, ) from superset.extensions import event_logger, security_manager from superset.models.dashboard import Dashboard @@ -135,6 +142,12 @@ from superset.subjects.filters import ( FilterRelatedSubjects, subject_type_filter, ) +from superset.tasks.export_dashboard_excel import ( + export_dashboard_excel, + EXPORT_LOCK_NAMESPACE, + export_lock_params, + EXPORT_LOCK_TTL_SECONDS, +) from superset.tasks.thumbnails import ( cache_dashboard_screenshot, cache_dashboard_thumbnail, @@ -274,6 +287,7 @@ class DashboardRestApi( "put_chart_customizations", "put_colors", "export_as_example", + "export_xlsx", "list_versions", "get_version", "activity", @@ -292,6 +306,10 @@ class DashboardRestApi( method_permission_name = { **MODEL_API_RW_METHOD_PERMISSION_MAP, "restore": "write", + # Reuse the dashboard ``can_export`` permission (the frontend gates the + # menu item on it) instead of the ``can_export_xlsx`` FAB would otherwise + # derive from the method name. + "export_xlsx": "export", } # Default list_columns (used if config not set) @@ -497,6 +515,8 @@ class DashboardRestApi( DashboardCopySchema, DashboardGetResponseSchema, DashboardDatasetSchema, + DashboardExportXlsxPostSchema, + DashboardExportXlsxResponseSchema, TabsPayloadSchema, GetFavStarIdsSchema, EmbeddedDashboardResponseSchema, @@ -1575,6 +1595,132 @@ class DashboardRestApi( response.set_cookie(token, "done", max_age=600) return response + @expose("//export_xlsx/", methods=("POST",)) + @protect() + @safe + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.export_xlsx", + log_to_statsd=False, + ) + def export_xlsx(self, pk: int) -> WerkzeugResponse: + """Export all of a dashboard's chart data to an Excel workbook (async). + --- + post: + summary: Export dashboard chart data to Excel + description: >- + Enqueues an async task that writes each chart's data to its own + worksheet, uploads the .xlsx to S3, and emails the requesting user a + pre-signed download link. Returns immediately with a job id. + parameters: + - in: path + schema: + type: integer + name: pk + description: The dashboard id + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DashboardExportXlsxPostSchema' + responses: + 202: + description: Export task accepted + content: + application/json: + schema: + $ref: '#/components/schemas/DashboardExportXlsxResponseSchema' + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 403: + $ref: '#/components/responses/403' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + 501: + description: Excel export is not configured on this server + """ + if not current_app.config["EXCEL_EXPORT_S3_BUCKET"]: + return self.response( + 501, message="Excel export is not configured on this server." + ) + try: + # Tolerate an empty/non-JSON body (e.g. a POST with no Content-Type); + # request.json would otherwise raise 415. + payload = DashboardExportXlsxPostSchema().load( + request.get_json(silent=True) or {} + ) + except ValidationError as error: + return self.response_400(message=error.messages) + + # Image export drives the headless webdriver, so it is only available + # when the same screenshot flags the UI checks are enabled. The decorator + # form (``@validate_feature_flags``) can't be used here because it would + # also block ``mode="data"``; mirror its 404 behavior inline instead. + if payload.get("mode") == "images" and not ( + is_feature_enabled("ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS") + and is_feature_enabled("ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT") + ): + return self.response_404() + + dashboard = cast(Dashboard, self.datamodel.get(pk, self._base_filters)) + if not dashboard: + return self.response_404() + try: + security_manager.raise_for_access(dashboard=dashboard) + except SupersetSecurityException: + return self.response_403() + + # Email delivery is the only result channel, so an account with an email + # address is required; embedded guest users are excluded in this version. + if isinstance(g.user, GuestUser) or not getattr(g.user, "email", None): + return self.response_400( + message="Excel export requires an account with an email address." + ) + if not dashboard.slices: + return self.response_400(message="Dashboard has no charts to export.") + + # Throttle: one concurrent export per user+dashboard. Acquire a shared, + # atomic distributed lock (Redis when configured, the metadata DB + # otherwise) so the guard works across the web server and workers and is + # not a no-op under the default cache. The task releases it when it + # settles; the TTL is the backstop if that release is ever lost. + lock_params = export_lock_params(g.user.id, dashboard.id) + try: + AcquireDistributedLock( + EXPORT_LOCK_NAMESPACE, + lock_params, + ttl_seconds=EXPORT_LOCK_TTL_SECONDS, + ).run() + except LockAlreadyHeldException: + return self.response( + 202, + message="An Excel export for this dashboard is already in progress.", + ) + + job_id = str(uuid.uuid4()) + try: + export_dashboard_excel.apply_async( + kwargs={ + "dashboard_id": dashboard.id, + "user_id": g.user.id, + "active_data_mask": payload.get("active_data_mask", {}), + "job_id": job_id, + "mode": payload.get("mode", "data"), + }, + task_id=job_id, + ) + except Exception: + # If enqueuing fails (e.g. broker down) the task will never run to + # release the lock, so free it now rather than block exports until + # the TTL expires. + ReleaseDistributedLock(EXPORT_LOCK_NAMESPACE, lock_params).run() + raise + return self.response(202, job_id=job_id) + @expose("//cache_dashboard_screenshot/", methods=("POST",)) @validate_feature_flags(["THUMBNAILS", "ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS"]) @protect() diff --git a/superset/dashboards/excel_export/__init__.py b/superset/dashboards/excel_export/__init__.py new file mode 100644 index 00000000000..13a83393a91 --- /dev/null +++ b/superset/dashboards/excel_export/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/superset/dashboards/excel_export/email.py b/superset/dashboards/excel_export/email.py new file mode 100644 index 00000000000..19233c0ac34 --- /dev/null +++ b/superset/dashboards/excel_export/email.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Email rendering and delivery for dashboard Excel exports. + +Bodies use inline styles only (no external CSS, no logo) to match Superset's +existing report notification emails, and all user-controlled values (dashboard +title, chart names) are HTML-escaped to avoid injection. +""" + +from __future__ import annotations + +from datetime import datetime + +from flask import current_app +from flask_babel import gettext as __, ngettext +from markupsafe import escape + +from superset.utils.core import send_email_smtp + +_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +_FOOTER_STYLE = "color:#888;font-size:12px;" +_BUTTON_STYLE = ( + "display:inline-block;padding:10px 16px;background:#20a7c9;color:#ffffff;" + "text-decoration:none;border-radius:4px;" +) + +# Reason keys under which the export task groups charts it could not export. +# The task classifies each omitted chart under one of these; the email renders a +# separate, labelled section per non-empty group with its own remediation text. +ERROR_NO_QUERY_CONTEXT = "no-query-context" +ERROR_GENERAL = "general-exception" + + +def _fmt(dt: datetime) -> str: + return dt.strftime(_DATETIME_FORMAT) + + +def _humanize_ttl(seconds: int) -> str: + """Render a TTL as a human-readable, pluralized, translatable duration. + + Whole hours read as "24 hours"; sub-hour and non-hour values keep their + minutes (e.g. "1 hour 30 minutes", "15 minutes") so the stated lifetime + always matches the real pre-signed URL expiration. + """ + hours, remainder = divmod(seconds, 3600) + parts: list[str] = [] + if hours: + parts.append(ngettext("%(num)d hour", "%(num)d hours", hours)) + if minutes := remainder // 60: + parts.append(ngettext("%(num)d minute", "%(num)d minutes", minutes)) + if not parts: + parts.append(ngettext("%(num)d second", "%(num)d seconds", seconds)) + return " ".join(parts) + + +def build_subject(dashboard_title: str, *, success: bool) -> str: + """Build the email subject, prefixed with EMAIL_REPORTS_SUBJECT_PREFIX.""" + prefix = current_app.config["EMAIL_REPORTS_SUBJECT_PREFIX"] + if success: + return prefix + __( + "Your dashboard export is ready: %(title)s", title=dashboard_title + ) + return prefix + __( + "Your dashboard export could not be completed: %(title)s", + title=dashboard_title, + ) + + +def _errored_section(errored: dict[str, list[str]]) -> str: + """Render one labelled, translated sub-list per non-empty error group. + + ``errored`` maps a reason key (see the ``ERROR_*`` constants) to the labels + of the charts that were omitted for that reason. Known reasons are rendered + first, in a stable order, each with its own remediation text; any unknown + reason key falls back to a generic message so nothing is silently dropped. + """ + if not errored: + return "" + notes = { + ERROR_NO_QUERY_CONTEXT: __( + "The following charts were omitted because they have no saved query " + "context. To include them, open each chart in Explore and re-save." + ), + ERROR_GENERAL: __( + "The following charts were omitted because an error occurred while " + "exporting them:" + ), + } + fallback = __("The following charts could not be exported:") + ordered = [ERROR_NO_QUERY_CONTEXT, ERROR_GENERAL] + reasons = ordered + [reason for reason in errored if reason not in ordered] + sections = [] + for reason in reasons: + labels = errored.get(reason) + if not labels: + continue + note = notes.get(reason, fallback) + items = "".join(f"
  • {escape(label)}
  • " for label in labels) + sections.append(f"

    {note}

      {items}
    ") + return "".join(sections) + + +def build_success_email( + dashboard_title: str, + download_url: str, + requested_at: datetime, + expires_at: datetime, + ttl_seconds: int, + errored: dict[str, list[str]], +) -> str: + """Render the success email body (HTML).""" + title = escape(dashboard_title) + url = escape(download_url) + ready = __('Your export of "%(title)s" is ready.', title=title) + button = __("Download Excel file") + expiry = __( + "This link expires in %(duration)s (%(when)s UTC).", + duration=_humanize_ttl(ttl_seconds), + when=_fmt(expires_at), + ) + requested = __( + "This export was requested on %(when)s UTC.", when=_fmt(requested_at) + ) + disclaimer = __("If you did not request this, you can ignore this email.") + return ( + '' + f"

    {ready}

    " + f'

    {button}

    ' + f"

    {expiry}

    " + f"{_errored_section(errored)}" + "
    " + f'

    {requested}
    {disclaimer}

    ' + "" + ) + + +def build_failure_email(dashboard_title: str, requested_at: datetime) -> str: + """Render the failure email body (HTML).""" + title = escape(dashboard_title) + failed = __('Your export of "%(title)s" could not be completed.', title=title) + advice = __( + "An error occurred while generating the file. Please try again, or " + "contact your administrator if the problem persists." + ) + requested = __( + "This export was requested on %(when)s UTC.", when=_fmt(requested_at) + ) + return ( + '' + f"

    {failed}

    " + f"

    {advice}

    " + "
    " + f'

    {requested}

    ' + "" + ) + + +def send_export_email(to: str, subject: str, html_content: str) -> None: + """Send an export email via the configured SMTP transport.""" + send_email_smtp( + to=to, + subject=subject, + html_content=html_content, + config=current_app.config, + ) diff --git a/superset/dashboards/excel_export/layout.py b/superset/dashboards/excel_export/layout.py new file mode 100644 index 00000000000..a18299b2fb3 --- /dev/null +++ b/superset/dashboards/excel_export/layout.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Determine the order in which a dashboard's charts appear in its layout.""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from superset.models.dashboard import Dashboard + from superset.models.slice import Slice + +CHART_TYPE = "CHART" +ROOT_ID = "ROOT_ID" + + +def _walk_chart_ids(position: dict[str, Any]) -> list[int]: + """ + Depth-first walk of a dashboard ``position_json`` returning chart ids in + visual (layout) order, including tab-nested charts. Each chart id appears + once (first occurrence wins); cycles are guarded against. + """ + if ROOT_ID not in position: + return [] + + ordered: list[int] = [] + seen_charts: set[int] = set() + visited_nodes: set[str] = set() + stack: list[str] = [ROOT_ID] + + while stack: + node_id = stack.pop() + if node_id in visited_nodes: + continue + visited_nodes.add(node_id) + + node = position.get(node_id) + if not isinstance(node, dict): + continue + + if node.get("type") == CHART_TYPE: + chart_id = node.get("meta", {}).get("chartId") + if isinstance(chart_id, int) and chart_id not in seen_charts: + seen_charts.add(chart_id) + ordered.append(chart_id) + + # Push children in reverse so they are popped in their declared order. + children = node.get("children", []) + for child_id in reversed(children): + stack.append(child_id) + + return ordered + + +def get_charts_in_layout_order(dashboard: Dashboard) -> list[Slice]: + """ + Return the dashboard's charts ordered by their position in the layout. + + Charts are visited depth-first over ``position_json`` (so tab-nested charts + are included in tab order), de-duplicated when the same chart is placed more + than once, and any chart that belongs to the dashboard but is absent from + the layout is appended at the end ordered by id. Layout entries that no + longer correspond to a dashboard chart are skipped. + + :param dashboard: The dashboard whose charts to order + :returns: The dashboard's :class:`Slice` objects in layout order + """ + slices_by_id: dict[int, Slice] = {slc.id: slc for slc in dashboard.slices} + + result: list[Slice] = [] + used: set[int] = set() + for chart_id in _walk_chart_ids(dashboard.position): + slc = slices_by_id.get(chart_id) + if slc is not None and chart_id not in used: + used.add(chart_id) + result.append(slc) + + orphans = sorted( + (slc for chart_id, slc in slices_by_id.items() if chart_id not in used), + key=lambda slc: slc.id, + ) + result.extend(orphans) + return result diff --git a/superset/dashboards/excel_export/screenshot.py b/superset/dashboards/excel_export/screenshot.py new file mode 100644 index 00000000000..c6c30759fa8 --- /dev/null +++ b/superset/dashboards/excel_export/screenshot.py @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Render a single dashboard chart to a PNG for the image-mode Excel export. + +This reuses the same headless render path scheduled reports use +(:class:`~superset.utils.screenshots.ChartScreenshot`), but points it at an +Explore URL whose ``form_data`` carries the live dashboard filter state — so an +embedded image reflects the same filters the data path applies, rather than the +chart's default saved state. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from celery.exceptions import SoftTimeLimitExceeded +from flask import current_app + +from superset.charts.data.dashboard_filter_context import ( + get_dashboard_filter_context, +) +from superset.utils import json +from superset.utils.screenshots import ChartScreenshot +from superset.utils.urls import get_url_path + +logger = logging.getLogger(__name__) + + +def render_chart_image( + chart: Any, + dashboard_id: int, + active_data_mask: dict[str, Any], + user: Any, +) -> bytes | None: + """ + Render ``chart`` (as seen on ``dashboard_id``) to PNG bytes. + + The chart is rendered through Explore in standalone mode with the live + dashboard filter state injected as ``extra_form_data`` — the same object the + data path merges into the query context — so the image and the data stay + consistent. + + :param chart: The ``Slice`` to render + :param dashboard_id: The dashboard the chart is displayed on (for filter scope) + :param active_data_mask: Live dashboard filter state keyed by native filter id + :param user: The requesting user; the render runs with their permissions + :returns: PNG bytes, or ``None`` if the render failed (the caller skips and + notes the chart) + """ + try: + filter_context = get_dashboard_filter_context( + dashboard_id=dashboard_id, + chart_id=chart.id, + active_data_mask=active_data_mask, + ) + + # Start from the chart's saved form data and force the slice id, then + # layer the live filters on top so the render matches the data path. + form_data: dict[str, Any] = json.loads(chart.params or "{}") + form_data["slice_id"] = chart.id + if filter_context.extra_form_data: + form_data["extra_form_data"] = filter_context.extra_form_data + + url = get_url_path( + "ExploreView.root", + form_data=json.dumps(form_data), + ) + + window_size = current_app.config["WEBDRIVER_WINDOW"]["slice"] + screenshot = ChartScreenshot( + url, + chart.digest, + window_size=window_size, + thumb_size=window_size, + ) + return screenshot.get_screenshot(user=user) + except SoftTimeLimitExceeded: + # A soft timeout aborts the whole export; don't let the broad handler + # below turn it into a ``None`` (a per-chart "could not render") result. + raise + except Exception: # pylint: disable=broad-except + logger.exception( + "Failed to render image for chart %s in dashboard %s", + getattr(chart, "id", "?"), + dashboard_id, + ) + return None diff --git a/superset/dashboards/schemas.py b/superset/dashboards/schemas.py index d0ffd31b285..b6e60f6224e 100644 --- a/superset/dashboards/schemas.py +++ b/superset/dashboards/schemas.py @@ -18,7 +18,7 @@ import re from typing import Any, Mapping, Union from marshmallow import fields, post_dump, post_load, pre_load, Schema -from marshmallow.validate import Length, ValidationError +from marshmallow.validate import Length, OneOf, ValidationError from superset import security_manager from superset.subjects.schemas import SubjectResponseSchema @@ -628,3 +628,30 @@ class CacheScreenshotSchema(Schema): fields.List(fields.Str(), validate=lambda x: len(x) == 2), required=False ) permalinkKey = fields.Str(required=False) # noqa: N815 + + +class DashboardExportXlsxPostSchema(Schema): + active_data_mask = fields.Dict( + keys=fields.Str(), + values=fields.Dict(), + load_default=dict, + metadata={ + "description": "Live dashboard filter state keyed by native filter id, " + "each carrying an extraFormData object." + }, + ) + mode = fields.String( + load_default="data", + validate=OneOf(["data", "images"]), + metadata={ + "description": "Export mode: 'data' streams each chart's tabular result " + "(default); 'images' embeds non-table charts as rendered images and " + "keeps table charts tabular." + }, + ) + + +class DashboardExportXlsxResponseSchema(Schema): + job_id = fields.String( + metadata={"description": "Correlation id for the async export task"} + ) diff --git a/superset/tasks/export_dashboard_excel.py b/superset/tasks/export_dashboard_excel.py new file mode 100644 index 00000000000..a5f6035e024 --- /dev/null +++ b/superset/tasks/export_dashboard_excel.py @@ -0,0 +1,361 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Celery task that exports every chart on a dashboard to a single multi-sheet +``.xlsx`` file, uploads it to S3, and emails the requesting user a pre-signed +download link. + +In ``"data"`` mode the task re-runs each chart's saved query context under the +requesting user, applies the live dashboard filter state, and streams the results +row-by-row into a constant-memory workbook so large dashboards never load all +data at once. In ``"images"`` mode non-table charts are instead rendered to +images (through the same headless path as scheduled reports, reflecting the live +filters) and embedded, while table-like charts stay tabular. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from datetime import datetime, timedelta, timezone +from typing import Any + +from celery.exceptions import SoftTimeLimitExceeded +from flask import current_app, g + +from superset import db, security_manager +from superset.charts.data.dashboard_filter_context import ( + apply_dashboard_filter_context, + get_dashboard_filter_context, +) +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.commands.distributed_lock.release import ReleaseDistributedLock +from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType +from superset.dashboards.excel_export import email +from superset.dashboards.excel_export.layout import get_charts_in_layout_order +from superset.dashboards.excel_export.screenshot import render_chart_image +from superset.extensions import celery_app +from superset.utils import json, s3 +from superset.utils.core import override_user +from superset.utils.excel_streaming import StreamingXlsxWriter + +logger = logging.getLogger(__name__) + +# Export modes: "data" streams every chart's tabular result (the default, +# unchanged behavior); "images" embeds non-table charts as rendered images and +# keeps only table-like charts tabular. +EXPORT_MODE_DATA = "data" +EXPORT_MODE_IMAGES = "images" + +# Viz types kept as tabular data in image mode; everything else is rendered as an +# image. Operators can override the set via ``EXCEL_EXPORT_TABLE_VIZ_TYPES``. +TABLE_VIZ_TYPES = {"table", "pivot_table_v2", "pivot_table"} + +EXPORT_SOFT_TIME_LIMIT = 600 +EXPORT_HARD_TIME_LIMIT = 660 + +# Namespace + TTL for the per-user+dashboard in-flight lock the API acquires +# before enqueue and this task releases when it settles. The lock uses the +# shared, atomic DistributedLock backend (Redis when configured, the metadata +# DB otherwise) so it actually synchronizes across the web server and workers — +# unlike a plain cache, which is a no-op under the default ``NullCache``. +# The TTL outlives the hard time limit so a worker killed at that limit (which +# skips the ``finally`` release) cannot hold the lock forever; the release in +# ``finally`` is the fast path that frees it as soon as the task settles. +EXPORT_LOCK_NAMESPACE = "excel_export" +EXPORT_LOCK_TTL_SECONDS = EXPORT_HARD_TIME_LIMIT + 60 + + +def export_lock_params(user_id: int, dashboard_id: int) -> dict[str, int]: + """Key parameters identifying the per-user+dashboard in-flight lock.""" + return {"user_id": user_id, "dashboard_id": dashboard_id} + + +class _ChartSkippedError(Exception): + """Signals a chart that could not be exported and should be listed as skipped.""" + + +def _chart_label(chart: Any) -> str: + """Human-readable label for a chart in the skipped-charts list.""" + return f"{chart.id} - {chart.slice_name or ''}".strip() + + +def _record_to_row(record: dict[str, Any], colnames: list[str]) -> list[Any]: + return [record.get(col) for col in colnames] + + +def _table_viz_types() -> set[str]: + """Viz types kept tabular in image mode (config override or built-in default).""" + return current_app.config.get("EXCEL_EXPORT_TABLE_VIZ_TYPES") or TABLE_VIZ_TYPES + + +def _renders_as_image(chart: Any, mode: str) -> bool: + """Whether this chart is embedded as an image rather than streamed as data.""" + return mode == EXPORT_MODE_IMAGES and chart.viz_type not in _table_viz_types() + + +def _write_chart_image_sheet( + writer: StreamingXlsxWriter, + chart: Any, + dashboard_id: int, + active_data_mask: dict[str, Any], + user: Any, +) -> None: + """ + Render a single chart to an image and embed it as its own sheet. + + :raises _ChartSkippedError: if the chart could not be rendered + """ + image = render_chart_image(chart, dashboard_id, active_data_mask, user) + if image is None: + raise _ChartSkippedError + writer.add_image_sheet(_chart_label(chart), image) + + +def _write_chart_sheets( + writer: StreamingXlsxWriter, + chart: Any, + dashboard_id: int, + active_data_mask: dict[str, Any], +) -> None: + """ + Run a single chart's query and stream its result(s) into the workbook. + + Charts may yield more than one query (e.g. mixed-series charts); each becomes + its own sheet. Raises if the chart cannot be exported, so the caller can skip + it and note it in the email. + """ + json_body = json.loads(chart.query_context) + # Override any stale saved values: we always want full JSON results. + json_body["result_format"] = ChartDataResultFormat.JSON + json_body["result_type"] = ChartDataResultType.FULL + json_body.pop("force", None) + + filter_context = get_dashboard_filter_context( + dashboard_id=dashboard_id, + chart_id=chart.id, + active_data_mask=active_data_mask, + ) + if filter_context.extra_form_data: + apply_dashboard_filter_context(json_body, filter_context.extra_form_data) + + # Jinja macros resolve form data from g.form_data; expose the saved context. + g.form_data = json_body + + query_context = ChartDataQueryContextSchema().load(json_body) + command = ChartDataCommand(query_context) + command.validate() + result = command.run() + + for index, query in enumerate(result["queries"]): + colnames = query.get("colnames") or [] + data = query.get("data") or [] + if index == 0: + name = f"{chart.id} - {chart.slice_name or ''}" + else: + name = f"{chart.id}.{index} - {chart.slice_name or ''}" + writer.add_sheet( + name, + colnames, + (_record_to_row(record, colnames) for record in data), + ) + + +def _build_workbook( + path: str, + dashboard: Any, + active_data_mask: dict[str, Any], + job_id: str, + mode: str, + user: Any, +) -> dict[str, list[str]]: + """Build the workbook on disk. + + Return the charts that could not be exported, grouped by the reason they + were omitted (see the ``email.ERROR_*`` reason keys), so the notification + can explain each group separately. + """ + errored: dict[str, list[str]] = {} + writer = StreamingXlsxWriter(path) + try: + for chart in get_charts_in_layout_order(dashboard): + label = _chart_label(chart) + as_image = _renders_as_image(chart, mode) + # Image charts render from their saved params and don't need a query + # context; data (and table) charts still do. + if not as_image and not chart.query_context: + errored.setdefault(email.ERROR_NO_QUERY_CONTEXT, []).append(label) + continue + try: + if as_image: + _write_chart_image_sheet( + writer, chart, dashboard.id, active_data_mask, user + ) + else: + _write_chart_sheets(writer, chart, dashboard.id, active_data_mask) + except SoftTimeLimitExceeded: + # A soft timeout is a task-level signal, not a per-chart failure: + # let it propagate so the outer handler emails a failure and runs + # cleanup, rather than continuing until the hard limit kills the + # worker (which would skip cleanup, leak temp files, and hold the + # in-flight lock until its TTL). ``except Exception`` below would + # otherwise swallow it, since it subclasses ``Exception``. + raise + except _ChartSkippedError: + logger.warning( + "Skipping chart %s in dashboard export %s (could not render)", + chart.id, + job_id, + ) + errored.setdefault(email.ERROR_GENERAL, []).append(label) + except Exception: # pylint: disable=broad-except + logger.exception( + "Skipping chart %s in dashboard export %s", chart.id, job_id + ) + errored.setdefault(email.ERROR_GENERAL, []).append(label) + + if writer.sheet_count == 0: + flat = [label for labels in errored.values() for label in labels] + writer.add_summary_sheet( + "Export Summary", + ["No chart data could be exported.", *flat], + ) + finally: + writer.close() + return errored + + +def _send_failure_email( + user: Any, dashboard_title: str, requested_at: datetime +) -> None: + if not (user and getattr(user, "email", None)): + return + try: + email.send_export_email( + user.email, + email.build_subject(dashboard_title, success=False), + email.build_failure_email(dashboard_title, requested_at), + ) + except Exception: # pylint: disable=broad-except + logger.exception("Failed to send export failure email") + + +@celery_app.task( + name="export_dashboard_excel", + bind=True, + soft_time_limit=EXPORT_SOFT_TIME_LIMIT, + time_limit=EXPORT_HARD_TIME_LIMIT, + max_retries=0, +) +def export_dashboard_excel( + self: Any, # pylint: disable=unused-argument + dashboard_id: int, + user_id: int, + active_data_mask: dict[str, Any], + job_id: str, + mode: str = EXPORT_MODE_DATA, +) -> None: + """ + Export a dashboard's charts to an ``.xlsx`` and email a download link. + + :param dashboard_id: The dashboard to export + :param user_id: The requesting user (the task runs with their permissions) + :param active_data_mask: Live dashboard filter state keyed by native filter id + :param job_id: Correlation id, also the Celery task id and S3 object name + :param mode: ``"data"`` streams every chart's tabular result; ``"images"`` + embeds non-table charts as rendered images and keeps tables tabular + """ + # pylint: disable=import-outside-toplevel + from superset.models.dashboard import Dashboard + + requested_at = datetime.now(tz=timezone.utc) + user = security_manager.get_user_by_id(user_id) + dashboard_title = "" + tmp_path: str | None = None + + try: + with override_user(user, force=False): + dashboard = ( + db.session.query(Dashboard).filter_by(id=dashboard_id).one_or_none() + ) + if dashboard is None: + raise ValueError(f"Dashboard {dashboard_id} not found") + dashboard_title = dashboard.dashboard_title or f"Dashboard {dashboard_id}" + + file_descriptor, tmp_path = tempfile.mkstemp( + suffix=".xlsx", prefix=f"dash-export-{job_id}-" + ) + os.close(file_descriptor) + + errored = _build_workbook( + tmp_path, dashboard, active_data_mask, job_id, mode, user + ) + + bucket = current_app.config["EXCEL_EXPORT_S3_BUCKET"] + key = ( + f"{current_app.config['EXCEL_EXPORT_S3_KEY_PREFIX']}" + f"{dashboard_id}/{job_id}.xlsx" + ) + ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"] + + s3.upload_file_to_s3(tmp_path, bucket, key) + download_url = s3.generate_presigned_url(bucket, key, ttl) + expires_at = datetime.now(tz=timezone.utc) + timedelta(seconds=ttl) + + if user and getattr(user, "email", None): + try: + email.send_export_email( + user.email, + email.build_subject(dashboard_title, success=True), + email.build_success_email( + dashboard_title=dashboard_title, + download_url=download_url, + requested_at=requested_at, + expires_at=expires_at, + ttl_seconds=ttl, + errored=errored, + ), + ) + except Exception: # pylint: disable=broad-except + # The file is already in S3; a send failure should not trigger + # a misleading failure email. + logger.exception("Failed to send export success email") + except SoftTimeLimitExceeded: + logger.warning("Dashboard excel export %s timed out", job_id) + _send_failure_email(user, dashboard_title, requested_at) + raise + except Exception: + logger.exception("Dashboard excel export %s failed", job_id) + _send_failure_email(user, dashboard_title, requested_at) + raise + finally: + try: + ReleaseDistributedLock( + EXPORT_LOCK_NAMESPACE, + export_lock_params(user_id, dashboard_id), + ).run() + except Exception: # pylint: disable=broad-except + # Best-effort: the lock's TTL is the backstop if this fails. + logger.exception( + "Failed to release in-flight export lock for user %s dashboard %s", + user_id, + dashboard_id, + ) + if tmp_path and os.path.exists(tmp_path): + os.remove(tmp_path) diff --git a/superset/utils/excel.py b/superset/utils/excel.py index 2af29e79a48..1a252de8fe5 100644 --- a/superset/utils/excel.py +++ b/superset/utils/excel.py @@ -41,18 +41,21 @@ NEUTRAL_DOCUMENT_PROPERTIES: dict[str, Any] = { "created": NEUTRAL_TIMESTAMP, } +# Leading characters that turn a cell into a formula in spreadsheet apps. Shared +# with the streaming writer (superset.utils.excel_streaming) so both export paths +# guard against the same formula-injection vectors. +FORMULA_PREFIXES = {"=", "+", "-", "@"} + def quote_formulas(df: pd.DataFrame) -> pd.DataFrame: """ Make sure to quote any formulas for security reasons. """ - formula_prefixes = {"=", "+", "-", "@"} - for col in df.select_dtypes(include="object").columns: df[col] = df[col].apply( lambda x: ( f"'{x}" - if isinstance(x, str) and len(x) and x[0] in formula_prefixes + if isinstance(x, str) and len(x) and x[0] in FORMULA_PREFIXES else x ) ) diff --git a/superset/utils/excel_streaming.py b/superset/utils/excel_streaming.py new file mode 100644 index 00000000000..c9d93403ea1 --- /dev/null +++ b/superset/utils/excel_streaming.py @@ -0,0 +1,250 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Streaming XLSX writer for multi-sheet dashboard exports. + +Unlike :mod:`superset.utils.excel`, which builds an in-memory DataFrame per +sheet and hands the whole thing to ``xlsxwriter`` at once, this writer opens the +workbook in ``constant_memory`` mode and writes rows one at a time, so +``xlsxwriter`` keeps at most one row per sheet buffered on the writer side. The +source records may still be materialized upstream (e.g. by the chart query +response); this bounds only the writer's own footprint, not the caller's. +""" + +from __future__ import annotations + +import math +import numbers +import re +from collections.abc import Iterable, Sequence +from datetime import date, datetime +from decimal import Decimal +from io import BytesIO +from typing import Any + +import xlsxwriter + +from superset.utils.excel import FORMULA_PREFIXES, NEUTRAL_DOCUMENT_PROPERTIES + +# Excel limits a sheet name to 31 characters and forbids these characters. +MAX_SHEET_NAME_LEN = 31 +_INVALID_SHEET_CHARS_RE = re.compile(r"[\[\]:*?/\\]") +# Excel reserves the sheet name "History" (case-insensitive). +_RESERVED_SHEET_NAME = "history" + +# A worksheet holds at most 1,048,576 rows; one is reserved for the header. +MAX_DATA_ROWS_PER_SHEET = 1_048_576 - 1 + +# Excel cannot represent integers beyond 10**15 without precision loss. +_MAX_EXCEL_INT = 10**15 + + +def _quote_if_formula(text: str) -> str: + """ + Prefix formula-like text with an apostrophe so spreadsheet apps treat it as + literal text (defense against formula injection). + + Leading whitespace is ignored when detecting a formula, because spreadsheet + apps still evaluate a cell whose formula prefix is preceded by spaces or + tabs (e.g. ``" =cmd"`` or ``"\\t=cmd"``). + """ + stripped = text.lstrip() + return f"'{text}" if stripped and stripped[0] in FORMULA_PREFIXES else text + + +def _coerce_float_cell(value: Any) -> Any: + """ + Convert a ``Decimal``/real value to something ``xlsxwriter`` accepts. + + ``float()`` on a non-finite ``Decimal`` ("NaN"/"Infinity") yields a value + xlsxwriter rejects, and an over-large value can raise ``OverflowError``; + blank the former and stringify the latter, and stringify magnitudes Excel + cannot represent precisely. + """ + try: + number = float(value) + except (OverflowError, ValueError): + return str(value) + if not math.isfinite(number): + return "" + return str(number) if abs(number) > _MAX_EXCEL_INT else number + + +def sanitize_sheet_name(raw: str, used: set[str]) -> str: + """ + Produce a valid, unique Excel sheet name from ``raw``. + + Replaces forbidden characters, strips surrounding apostrophes/whitespace, + avoids the reserved name "History", truncates to 31 characters, and + disambiguates case-insensitive collisions with ``~2``/``~3`` suffixes. + The chosen name (lower-cased) is added to ``used``. + + :param raw: The desired sheet name (e.g. ``"42 - Sales by Region"``) + :param used: Lower-cased names already taken; mutated with the result + :returns: A sanitized, unique sheet name no longer than 31 characters + """ + name = _INVALID_SHEET_CHARS_RE.sub("_", raw or "") + name = name.strip().strip("'").strip() + if not name: + name = "Sheet" + if name.lower() == _RESERVED_SHEET_NAME: + name = f"{name}_" + name = name[:MAX_SHEET_NAME_LEN] + + if name.lower() not in used: + used.add(name.lower()) + return name + + suffix = 2 + while True: + marker = f"~{suffix}" + candidate = name[: MAX_SHEET_NAME_LEN - len(marker)] + marker + if candidate.lower() not in used: + used.add(candidate.lower()) + return candidate + suffix += 1 + + +def _sanitize_cell(value: Any) -> Any: + """ + Coerce a single cell value into something safe for ``xlsxwriter``. + + Quotes formula-like strings (defense against formula injection), stringifies + integers/floats Excel cannot represent precisely, renders temporal values as + ISO strings (timezones are not natively supported), and blanks out ``None`` + and non-finite floats. + """ + if value is None: + return "" + # bool is a subclass of int; preserve it before the numeric branches. + if isinstance(value, bool): + return value + if isinstance(value, str): + return _quote_if_formula(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Decimal): + return _coerce_float_cell(value) + if isinstance(value, numbers.Integral): + number = int(value) + return str(number) if abs(number) > _MAX_EXCEL_INT else number + if isinstance(value, numbers.Real): + return _coerce_float_cell(value) + # Anything else (lists, dicts, custom objects) is stringified, still guarding + # against formula injection on the resulting text. + return _quote_if_formula(str(value)) + + +class StreamingXlsxWriter: + """ + A thin wrapper over ``xlsxwriter`` in constant-memory mode that writes one + sheet per chart, row by row. + + Sheet names are sanitized and de-duplicated, cell values are sanitized for + safety/compatibility, and per-sheet row counts are capped at Excel's limit. + Always call :meth:`close` (e.g. in a ``finally`` block) to finalize the file. + """ + + def __init__(self, path: str) -> None: + self._workbook = xlsxwriter.Workbook(path, {"constant_memory": True}) + # Reset document properties so the file carries no identifying details. + self._workbook.set_properties(NEUTRAL_DOCUMENT_PROPERTIES) + self._used_sheet_names: set[str] = set() + self.sheet_count = 0 + + def add_sheet( + self, + name: str, + columns: Sequence[Any], + rows: Iterable[Sequence[Any]], + ) -> int: + """ + Write a header row followed by data rows into a new sheet. + + :param name: Desired sheet name (sanitized/de-duplicated automatically) + :param columns: Column headers + :param rows: Iterable of row sequences, streamed one at a time + :returns: The number of data rows actually written (capped just below + Excel's per-sheet limit; when the data is larger a final notice row + is appended and the dropped rows are not counted) + """ + sheet_name = sanitize_sheet_name(name, self._used_sheet_names) + worksheet = self._workbook.add_worksheet(sheet_name) + worksheet.write_row(0, 0, [_sanitize_cell(col) for col in columns]) + + # Reserve the final row for a truncation notice, so when the data + # exceeds the sheet's capacity the user can see rows were dropped + # instead of silently losing them. + row_cap = MAX_DATA_ROWS_PER_SHEET - 1 + written = 0 + truncated = False + for row in rows: + if written >= row_cap: + truncated = True + break + worksheet.write_row(written + 1, 0, [_sanitize_cell(cell) for cell in row]) + written += 1 + + if truncated: + worksheet.write_string( + written + 1, + 0, + f"[Truncated: only first {written:,} rows exported]", + ) + + self.sheet_count += 1 + return written + + def add_image_sheet(self, name: str, image_bytes: bytes) -> None: + """ + Write a single sheet holding a rendered chart image. + + The image is embedded top-left; ``xlsxwriter`` buffers image data and + writes it at :meth:`close`, so this composes with ``constant_memory`` + mode just like the row-streaming sheets. + + :param name: Desired sheet name (sanitized/de-duplicated automatically) + :param image_bytes: PNG bytes to embed + """ + sheet_name = sanitize_sheet_name(name, self._used_sheet_names) + worksheet = self._workbook.add_worksheet(sheet_name) + worksheet.insert_image( + 0, + 0, + f"{sheet_name}.png", + {"image_data": BytesIO(image_bytes)}, + ) + self.sheet_count += 1 + + def add_summary_sheet(self, name: str, lines: Sequence[str]) -> None: + """ + Write a single-column informational sheet (e.g. a list of skipped charts). + + Lines are written as string cells, so formula-like text is never executed. + """ + sheet_name = sanitize_sheet_name(name, self._used_sheet_names) + worksheet = self._workbook.add_worksheet(sheet_name) + for index, line in enumerate(lines): + worksheet.write_string(index, 0, str(line)) + self.sheet_count += 1 + + def close(self) -> None: + """Finalize and write the workbook to disk.""" + if self.sheet_count == 0: + # Excel requires at least one worksheet for a valid file. + self._workbook.add_worksheet("Export") + self._workbook.close() diff --git a/superset/utils/s3.py b/superset/utils/s3.py new file mode 100644 index 00000000000..1e68b980964 --- /dev/null +++ b/superset/utils/s3.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Minimal S3 helpers for uploading export artifacts and minting pre-signed URLs. + +Credentials and region come from the standard boto3 resolution chain (env vars, +shared config, instance role). Operators can override client construction via +the ``EXCEL_EXPORT_S3_CLIENT_KWARGS`` config (e.g. ``region_name`` or an +``endpoint_url`` for S3-compatible stores such as MinIO/LocalStack). +""" + +from __future__ import annotations + +import logging +from typing import Any + +from flask import current_app + +logger = logging.getLogger(__name__) + + +def _get_s3_client() -> Any: + """Build an S3 client using operator-provided client kwargs (if any).""" + # boto3 is imported lazily so that importing this module (which happens at + # app startup via the dashboard API) does not require boto3 to be installed. + # The dependency is only needed when an export actually runs; if it is + # missing, surface an actionable install hint rather than a bare ImportError. + try: + import boto3 # pylint: disable=import-outside-toplevel + except ImportError as ex: + raise ImportError( + "boto3 is required for dashboard Excel export but is not installed. " + "Install it with `pip install apache-superset[excel-export]`." + ) from ex + + client_kwargs: dict[str, Any] = current_app.config.get( + "EXCEL_EXPORT_S3_CLIENT_KWARGS", {} + ) + return boto3.client("s3", **client_kwargs) + + +def upload_file_to_s3(local_path: str, bucket: str, key: str) -> None: + """ + Upload a local file to S3. + + ``boto3``'s ``upload_file`` automatically uses a managed multipart transfer + for large files, so no manual chunking is required. + + :param local_path: Path to the file on local disk + :param bucket: Destination S3 bucket + :param key: Destination S3 object key + """ + _get_s3_client().upload_file(local_path, bucket, key) + + +def generate_presigned_url(bucket: str, key: str, expires_in: int) -> str: + """ + Generate a time-limited pre-signed URL for downloading an S3 object. + + :param bucket: The S3 bucket + :param key: The S3 object key + :param expires_in: URL lifetime in seconds + :returns: A pre-signed ``get_object`` URL + """ + return _get_s3_client().generate_presigned_url( + "get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=expires_in, + ) diff --git a/tests/integration_tests/dashboards/api_tests.py b/tests/integration_tests/dashboards/api_tests.py index a87bb797d0c..a25af9fcd48 100644 --- a/tests/integration_tests/dashboards/api_tests.py +++ b/tests/integration_tests/dashboards/api_tests.py @@ -31,6 +31,7 @@ import yaml from freezegun import freeze_time from sqlalchemy import and_ from superset import db, security_manager # noqa: F401 +from superset.exceptions import LockAlreadyHeldException from superset.models.dashboard import Dashboard from superset.models.core import FavStar, FavStarClassName from superset.reports.models import ReportSchedule, ReportScheduleType @@ -42,6 +43,7 @@ from superset.utils.core import backend, override_user from superset.utils.screenshots import ScreenshotCachePayload from superset.utils import json +from tests.conftest import with_config from tests.integration_tests.base_api_tests import ApiEditorsTestCaseMixin from tests.integration_tests.base_tests import ( subjects_from_users, @@ -3262,6 +3264,173 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa response = json.loads(rv.data.decode("utf-8")) assert response["count"] > 0 + def test_export_xlsx_501_when_bucket_unset(self): + """Dashboard API: export_xlsx returns 501 when the S3 bucket is unset.""" + admin = self.get_user("admin") + dashboard = self.insert_dashboard("xlsx-501", None, [admin.id]) + self.login(ADMIN_USERNAME) + try: + rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/") + assert rv.status_code == 501 + finally: + db.session.delete(dashboard) + db.session.commit() + + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_404_for_missing_dashboard(self, mock_task): + """Dashboard API: export_xlsx returns 404 for an unknown dashboard.""" + self.login(ADMIN_USERNAME) + rv = self.client.post("api/v1/dashboard/99999999/export_xlsx/") + assert rv.status_code == 404 + mock_task.apply_async.assert_not_called() + + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_400_for_empty_dashboard(self, mock_task): + """Dashboard API: export_xlsx returns 400 for a dashboard with no charts.""" + admin = self.get_user("admin") + dashboard = self.insert_dashboard("xlsx-empty", None, [admin.id]) + self.login(ADMIN_USERNAME) + try: + rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/") + assert rv.status_code == 400 + mock_task.apply_async.assert_not_called() + finally: + db.session.delete(dashboard) + db.session.commit() + + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.AcquireDistributedLock") + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_202_enqueues_task(self, mock_task, mock_acquire): + """Dashboard API: export_xlsx enqueues the task and returns 202 + job_id.""" + self.login(ADMIN_USERNAME) + dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first() + rv = self.client.post( + f"api/v1/dashboard/{dashboard.id}/export_xlsx/", + json={"active_data_mask": {}}, + ) + assert rv.status_code == 202 + body = json.loads(rv.data.decode("utf-8")) + job_id = body["job_id"] + assert job_id + # The in-flight lock is acquired before the task is enqueued. + mock_acquire.return_value.run.assert_called_once() + mock_task.apply_async.assert_called_once() + _, kwargs = mock_task.apply_async.call_args + assert kwargs["task_id"] == job_id + assert kwargs["kwargs"]["dashboard_id"] == dashboard.id + + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.AcquireDistributedLock") + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_202_when_export_already_in_progress( + self, mock_task, mock_acquire + ): + """Dashboard API: export_xlsx does not enqueue a second concurrent export.""" + # An in-flight lock is already held for this user+dashboard. + mock_acquire.return_value.run.side_effect = LockAlreadyHeldException("held") + self.login(ADMIN_USERNAME) + dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first() + rv = self.client.post( + f"api/v1/dashboard/{dashboard.id}/export_xlsx/", + json={"active_data_mask": {}}, + ) + assert rv.status_code == 202 + assert "already in progress" in rv.data.decode("utf-8") + mock_task.apply_async.assert_not_called() + + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_404_for_inaccessible_dashboard(self, mock_task): + """Dashboard API: export_xlsx returns 404 for a dashboard the user can't see.""" + admin = self.get_user("admin") + dashboard = self.insert_dashboard( + "xlsx-private", None, [admin.id], published=False + ) + self.login(GAMMA_USERNAME) + try: + rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/") + assert rv.status_code == 404 + mock_task.apply_async.assert_not_called() + finally: + db.session.delete(dashboard) + db.session.commit() + + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.AcquireDistributedLock") + @patch("superset.dashboards.api.export_dashboard_excel") + @patch("superset.dashboards.api.security_manager.raise_for_access") + def test_export_xlsx_admitted_with_can_export_only( + self, mock_raise, mock_task, mock_acquire + ): + """Dashboard API: export_xlsx is gated on ``can_export``, not a distinct + ``can_export_xlsx``. Gamma holds dashboard ``can_export`` by default (and + the frontend shows the menu item on that basis), so a Gamma user must be + admitted (202) rather than rejected by ``@protect()`` (403).""" + gamma_user = security_manager.find_user(username=GAMMA_USERNAME) + slice_ = db.session.query(Slice).first() + # Clone Gamma (so the login password is valid); Gamma already carries + # dashboard ``can_export``. + with self.temporary_user(gamma_user, login=True) as user: + dashboard = self.insert_dashboard( + "xlsx-can-export", None, [user.id], slices=[slice_], published=True + ) + try: + rv = self.client.post( + f"api/v1/dashboard/{dashboard.id}/export_xlsx/", + json={"active_data_mask": {}}, + ) + assert rv.status_code == 202 + mock_task.apply_async.assert_called_once() + finally: + db.session.delete(dashboard) + db.session.commit() + + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_images_404_when_screenshot_flags_off(self, mock_task): + """Dashboard API: ``mode=images`` is rejected with 404 when the webdriver + screenshot flags are disabled (the same signal the UI gates the option on), + and no task is enqueued.""" + self.login(ADMIN_USERNAME) + dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first() + rv = self.client.post( + f"api/v1/dashboard/{dashboard.id}/export_xlsx/", + json={"active_data_mask": {}, "mode": "images"}, + ) + assert rv.status_code == 404 + mock_task.apply_async.assert_not_called() + + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") + @with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"}) + @with_feature_flags( + ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS=True, + ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT=True, + ) + @patch("superset.dashboards.api.AcquireDistributedLock") + @patch("superset.dashboards.api.export_dashboard_excel") + def test_export_xlsx_images_202_when_screenshot_flags_on( + self, mock_task, mock_acquire + ): + """Dashboard API: ``mode=images`` is accepted (202) when both webdriver + screenshot flags are enabled.""" + self.login(ADMIN_USERNAME) + dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first() + rv = self.client.post( + f"api/v1/dashboard/{dashboard.id}/export_xlsx/", + json={"active_data_mask": {}, "mode": "images"}, + ) + assert rv.status_code == 202 + mock_task.apply_async.assert_called_once() + _, kwargs = mock_task.apply_async.call_args + assert kwargs["kwargs"]["mode"] == "images" + @pytest.mark.usefixtures("load_world_bank_dashboard_with_slices") def test_embedded_dashboards(self): self.login(ADMIN_USERNAME) diff --git a/tests/unit_tests/charts/test_dashboard_filter_context.py b/tests/unit_tests/charts/test_dashboard_filter_context.py index 3bc062330bf..f784eeb744a 100644 --- a/tests/unit_tests/charts/test_dashboard_filter_context.py +++ b/tests/unit_tests/charts/test_dashboard_filter_context.py @@ -563,6 +563,178 @@ def test_get_dashboard_filter_context_out_of_scope_filter_excluded( assert ctx.filters[0].id == "f1" +def _build_dashboard_mock( + mock_db: MagicMock, + filter_config: list[dict[str, Any]], + chart_ids: list[int], +) -> MagicMock: + """Wire a dashboard MagicMock with the given filters and chart ids.""" + metadata = {"native_filter_configuration": filter_config} + dashboard = MagicMock() + dashboard.id = 1 + dashboard.slices = [MagicMock(id=cid) for cid in chart_ids] + dashboard.json_metadata = json.dumps(metadata) + dashboard.position_json = json.dumps(SAMPLE_POSITION_JSON) + ( + mock_db.session.query.return_value.filter_by.return_value.one_or_none.return_value + ) = dashboard + return dashboard + + +@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") +@patch("superset.charts.data.dashboard_filter_context.db") +def test_active_data_mask_overrides_default( + mock_db: MagicMock, + mock_check_access: MagicMock, +) -> None: + """An active filter value replaces the saved default.""" + filter_config = [ + _make_filter( + flt_id="f1", + name="Region", + scope_root=["ROOT_ID"], + default_value=["US"], + target_column="region", + ), + ] + _build_dashboard_mock(mock_db, filter_config, [10]) + + active_data_mask = { + "f1": { + "extraFormData": { + "filters": [{"col": "region", "op": "IN", "val": ["FR", "DE"]}] + } + } + } + ctx = get_dashboard_filter_context( + dashboard_id=1, chart_id=10, active_data_mask=active_data_mask + ) + + assert ctx.filters[0].status == DashboardFilterStatus.APPLIED + assert ctx.extra_form_data["filters"][0]["val"] == ["FR", "DE"] + + +@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") +@patch("superset.charts.data.dashboard_filter_context.db") +def test_active_data_mask_empty_clears_default( + mock_db: MagicMock, + mock_check_access: MagicMock, +) -> None: + """An empty active extraFormData clears the filter; the default is NOT used.""" + filter_config = [ + _make_filter( + flt_id="f1", + name="Region", + scope_root=["ROOT_ID"], + default_value=["US"], + target_column="region", + ), + ] + _build_dashboard_mock(mock_db, filter_config, [10]) + + active_data_mask: dict[str, Any] = {"f1": {"extraFormData": {}}} + ctx = get_dashboard_filter_context( + dashboard_id=1, chart_id=10, active_data_mask=active_data_mask + ) + + assert ctx.filters[0].status == DashboardFilterStatus.NOT_APPLIED + assert "filters" not in ctx.extra_form_data + + +@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") +@patch("superset.charts.data.dashboard_filter_context.db") +def test_active_data_mask_absent_filter_falls_back_to_default( + mock_db: MagicMock, + mock_check_access: MagicMock, +) -> None: + """A filter not present in the mask keeps its saved default.""" + filter_config = [ + _make_filter( + flt_id="f1", + name="Region", + scope_root=["ROOT_ID"], + default_value=["US"], + target_column="region", + ), + ] + _build_dashboard_mock(mock_db, filter_config, [10]) + + # Mask only references some other filter id + active_data_mask: dict[str, Any] = {"f2": {"extraFormData": {"filters": []}}} + ctx = get_dashboard_filter_context( + dashboard_id=1, chart_id=10, active_data_mask=active_data_mask + ) + + assert ctx.filters[0].status == DashboardFilterStatus.APPLIED + assert ctx.extra_form_data["filters"][0]["val"] == ["US"] + + +@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") +@patch("superset.charts.data.dashboard_filter_context.db") +def test_active_data_mask_applies_despite_default_to_first_item( + mock_db: MagicMock, + mock_check_access: MagicMock, +) -> None: + """ + defaultToFirstItem filters cannot be resolved from saved config, but when the + frontend supplies a concrete active value it is applied. + """ + filter_config = [ + _make_filter( + flt_id="f1", + name="City", + scope_root=["ROOT_ID"], + default_to_first_item=True, + target_column="city", + ), + ] + _build_dashboard_mock(mock_db, filter_config, [10]) + + active_data_mask = { + "f1": { + "extraFormData": {"filters": [{"col": "city", "op": "IN", "val": ["NYC"]}]} + } + } + ctx = get_dashboard_filter_context( + dashboard_id=1, chart_id=10, active_data_mask=active_data_mask + ) + + assert ctx.filters[0].status == DashboardFilterStatus.APPLIED + assert ctx.extra_form_data["filters"][0]["val"] == ["NYC"] + + +@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") +@patch("superset.charts.data.dashboard_filter_context.db") +def test_active_data_mask_out_of_scope_filter_still_excluded( + mock_db: MagicMock, + mock_check_access: MagicMock, +) -> None: + """An active value for an out-of-scope filter does not leak into the chart.""" + filter_config = [ + _make_filter( + flt_id="f1", + name="Out-of-scope", + scope_root=["TABS-nonexistent"], + target_column="status", + ), + ] + _build_dashboard_mock(mock_db, filter_config, [10]) + + active_data_mask = { + "f1": { + "extraFormData": { + "filters": [{"col": "status", "op": "IN", "val": ["active"]}] + } + } + } + ctx = get_dashboard_filter_context( + dashboard_id=1, chart_id=10, active_data_mask=active_data_mask + ) + + assert ctx.filters == [] + assert ctx.extra_form_data == {} + + @patch("superset.charts.data.dashboard_filter_context._check_dashboard_access") @patch("superset.charts.data.dashboard_filter_context.db") def test_get_dashboard_filter_context_chart_not_in_layout_receives_root_filters( diff --git a/tests/unit_tests/dashboards/test_excel_export_email.py b/tests/unit_tests/dashboards/test_excel_export_email.py new file mode 100644 index 00000000000..de9bc294842 --- /dev/null +++ b/tests/unit_tests/dashboards/test_excel_export_email.py @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime +from unittest.mock import MagicMock, patch + +from superset.dashboards.excel_export import email + +REQUESTED = datetime(2026, 1, 1, 12, 0, 0) +EXPIRES = datetime(2026, 1, 2, 12, 0, 0) + + +def test_success_email_contains_link_and_expiry() -> None: + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://signed.example/file.xlsx?sig=abc", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=86400, + errored={}, + ) + assert "https://signed.example/file.xlsx?sig=abc" in html + assert "expires in 24 hours" in html + assert "2026-01-02 12:00:00 UTC" in html + assert "2026-01-01 12:00:00 UTC" in html + assert "
  • " not in html # no skipped section + + +def test_success_email_sub_hour_ttl_reports_minutes() -> None: + # A sub-hour TTL must not truncate to "0 hours"; it should report minutes. + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=900, + errored={}, + ) + assert "expires in 15 minutes" in html + assert "0 hours" not in html + + +def test_success_email_mixed_ttl_reports_hours_and_minutes() -> None: + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=5400, + errored={}, + ) + assert "expires in 1 hour 30 minutes" in html + + +def test_success_email_lists_charts_with_no_query_context() -> None: + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=86400, + errored={email.ERROR_NO_QUERY_CONTEXT: ["10 - Broken chart"]}, + ) + assert "no saved query context" in html + assert "
  • 10 - Broken chart
  • " in html + + +def test_success_email_groups_errors_by_reason() -> None: + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=86400, + errored={ + email.ERROR_NO_QUERY_CONTEXT: ["10 - NoContext"], + email.ERROR_GENERAL: ["30 - Boom"], + }, + ) + # Each reason renders its own labelled section with the right chart. + assert "no saved query context" in html + assert "an error occurred" in html + assert "
  • 10 - NoContext
  • " in html + assert "
  • 30 - Boom
  • " in html + + +def test_success_email_omits_empty_reason_groups() -> None: + html = email.build_success_email( + dashboard_title="Sales", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=86400, + errored={email.ERROR_GENERAL: ["30 - Boom"]}, + ) + assert "an error occurred" in html + assert "no saved query context" not in html + assert "
  • 30 - Boom
  • " in html + + +def test_success_email_escapes_title() -> None: + html = email.build_success_email( + dashboard_title="", + download_url="https://x", + requested_at=REQUESTED, + expires_at=EXPIRES, + ttl_seconds=86400, + errored={}, + ) + assert "