mirror of
https://github.com/apache/superset.git
synced 2026-08-25 17:41:14 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e4ca732979 | ||
|
|
4e7f040a1a | ||
|
|
a42f5a2fa8 | ||
|
|
f94582e2ca | ||
|
|
8f5283fa0a | ||
|
|
a9e680cbe1 | ||
|
|
44413d8aee | ||
|
|
4423f5034b | ||
|
|
26fabcf77d |
@@ -184,6 +184,12 @@ excel = ["xlrd>=2.0.2, <2.1"]
|
||||
# emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so
|
||||
# installing this extra is only required to actually run exports.
|
||||
excel-export = ["boto3"]
|
||||
# Alternate dashboard Excel export storage backend (EXCEL_EXPORT_STORAGE =
|
||||
# GCSExportStorage()) for a deployment whose export bucket is a native Google
|
||||
# Cloud Storage bucket rather than S3. google-cloud-storage is imported lazily
|
||||
# by superset.utils.gcs, so this extra is an alternative to excel-export, not
|
||||
# an addition to it -- pick whichever matches your bucket's provider.
|
||||
excel-export-gcs = ["google-cloud-storage"]
|
||||
fastmcp = [
|
||||
"fastmcp>=3.4.6,<4.0",
|
||||
# tiktoken backs the response-size-guard token estimator. Without
|
||||
|
||||
+169
-4
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
import React from 'react';
|
||||
import {
|
||||
act,
|
||||
render,
|
||||
screen,
|
||||
userEvent,
|
||||
@@ -32,8 +33,20 @@ import {
|
||||
} from '@superset-ui/core';
|
||||
import { useDownloadMenuItems } from '.';
|
||||
|
||||
const mockRedirect = jest.fn();
|
||||
jest.mock('src/utils/navigationUtils', () => ({
|
||||
...jest.requireActual('src/utils/navigationUtils'),
|
||||
redirect: (url: string) => mockRedirect(url),
|
||||
}));
|
||||
|
||||
const mockAddSuccessToast = jest.fn();
|
||||
const mockAddDangerToast = jest.fn();
|
||||
const mockAddInfoToast = jest.fn();
|
||||
|
||||
let mockIsEmbedded = false;
|
||||
jest.mock('src/dashboard/util/isEmbedded', () => ({
|
||||
isEmbedded: () => mockIsEmbedded,
|
||||
}));
|
||||
|
||||
jest.mock('src/components/MessageToasts/withToasts', () => ({
|
||||
__esModule: true,
|
||||
@@ -41,6 +54,7 @@ jest.mock('src/components/MessageToasts/withToasts', () => ({
|
||||
useToasts: () => ({
|
||||
addSuccessToast: mockAddSuccessToast,
|
||||
addDangerToast: mockAddDangerToast,
|
||||
addInfoToast: mockAddInfoToast,
|
||||
}),
|
||||
}));
|
||||
|
||||
@@ -89,12 +103,17 @@ const MenuWrapperWithProps = (
|
||||
|
||||
const originalCreateObjectURL = window.URL.createObjectURL;
|
||||
const originalRevokeObjectURL = window.URL.revokeObjectURL;
|
||||
const originalLocation = window.location;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockIsEmbedded = false;
|
||||
// Reset the implementation each test: clearAllMocks resets call history but
|
||||
// not mockReturnValue, so an override in one test would otherwise leak.
|
||||
(isFeatureEnabled as jest.Mock).mockReturnValue(false);
|
||||
// @ts-ignore
|
||||
delete window.location;
|
||||
window.location = { href: '' } as Location;
|
||||
});
|
||||
|
||||
// "Export Images to Excel" is gated on the webdriver screenshot feature flags.
|
||||
@@ -104,6 +123,8 @@ const enableWebDriverScreenshot = () =>
|
||||
afterEach(() => {
|
||||
window.URL.createObjectURL = originalCreateObjectURL;
|
||||
window.URL.revokeObjectURL = originalRevokeObjectURL;
|
||||
window.location = originalLocation;
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('Should render all menu items', () => {
|
||||
@@ -155,8 +176,9 @@ test('Export Data to Excel posts mode "data" and shows a pending toast', async (
|
||||
endpoint: '/api/v1/dashboard/123/export_xlsx/',
|
||||
jsonPayload: { active_data_mask: {}, mode: 'data' },
|
||||
});
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
{ noDuplicate: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -176,12 +198,121 @@ test('Export Images to Excel posts mode "images" and shows a pending toast', asy
|
||||
endpoint: '/api/v1/dashboard/123/export_xlsx/',
|
||||
jsonPayload: { active_data_mask: {}, mode: 'images' },
|
||||
});
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
{ noDuplicate: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Data to Excel polls status and auto-downloads once ready', async () => {
|
||||
// A guest/embedded session has no email to be notified at, so completion is
|
||||
// discovered by polling export_xlsx/status/<job_id>/ instead -- exercised
|
||||
// here regardless of session type, since the same polling drives the
|
||||
// auto-download for a regular session too.
|
||||
jest.useFakeTimers();
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
mockSupersetClient.get.mockResolvedValue({
|
||||
json: {
|
||||
status: 'ready',
|
||||
download_url: '/api/v1/dashboard/export_xlsx/download/abc/',
|
||||
},
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
await waitFor(() =>
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
{ noDuplicate: true },
|
||||
),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
jest.advanceTimersByTime(3000);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockSupersetClient.get).toHaveBeenCalledWith({
|
||||
endpoint: '/api/v1/dashboard/export_xlsx/status/abc/',
|
||||
});
|
||||
expect(mockRedirect).toHaveBeenCalledWith(
|
||||
'/api/v1/dashboard/export_xlsx/download/abc/',
|
||||
);
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
'Your export is ready and downloading.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Data to Excel keeps polling while status is pending', async () => {
|
||||
jest.useFakeTimers();
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
mockSupersetClient.get.mockResolvedValue({
|
||||
json: { status: 'pending' },
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
await waitFor(() =>
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
{ noDuplicate: true },
|
||||
),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
jest.advanceTimersByTime(3000);
|
||||
});
|
||||
await waitFor(() => expect(mockSupersetClient.get).toHaveBeenCalledTimes(1));
|
||||
|
||||
await act(async () => {
|
||||
jest.advanceTimersByTime(3000);
|
||||
});
|
||||
await waitFor(() => expect(mockSupersetClient.get).toHaveBeenCalledTimes(2));
|
||||
|
||||
// Still pending -- no terminal toast, and the browser never navigated.
|
||||
expect(mockAddDangerToast).not.toHaveBeenCalled();
|
||||
expect(mockRedirect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('Export Data to Excel shows an error toast when the export job fails', async () => {
|
||||
jest.useFakeTimers();
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
mockSupersetClient.get.mockResolvedValue({
|
||||
json: { status: 'error', message: 'The export could not be built.' },
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
await waitFor(() =>
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
{ noDuplicate: true },
|
||||
),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
jest.advanceTimersByTime(3000);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockAddDangerToast).toHaveBeenCalledWith(
|
||||
'The export could not be built.',
|
||||
);
|
||||
});
|
||||
expect(mockRedirect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('Export Data to Excel shows an "already in progress" toast when throttled', async () => {
|
||||
// The throttle response is 202 with a message but no job_id.
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
@@ -368,3 +499,37 @@ test('Enabled screenshot items should not show tooltip icon', () => {
|
||||
|
||||
mockIsFeatureEnabled.mockReset();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Embedded (guest) behavior: no email channel exists, so the toast must not
|
||||
// promise one, and the image export (webdriver-rendered, guest cannot open
|
||||
// Explore) is hidden.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('embedded: export toast promises auto-download, not an email', async () => {
|
||||
mockIsEmbedded = true;
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockAddInfoToast).toHaveBeenCalledWith(
|
||||
'Your export is being generated. Please, do not leave the page.',
|
||||
{ noDuplicate: true },
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('embedded: Export Images to Excel is hidden even with the webdriver enabled', () => {
|
||||
mockIsEmbedded = true;
|
||||
enableWebDriverScreenshot();
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
expect(screen.getByText('Export Data to Excel')).toBeInTheDocument();
|
||||
expect(screen.queryByText('Export Images to Excel')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
@@ -29,11 +29,13 @@ import {
|
||||
import { MenuItem } from '@superset-ui/core/components/Menu';
|
||||
import { parse as parseContentDisposition } from 'content-disposition';
|
||||
import { useDownloadScreenshot } from 'src/dashboard/hooks/useDownloadScreenshot';
|
||||
import { isEmbedded as isEmbeddedDashboard } from 'src/dashboard/util/isEmbedded';
|
||||
import { NATIVE_FILTER_PREFIX } from 'src/dashboard/components/nativeFilters/FiltersConfigModal/utils';
|
||||
import { MenuKeys, RootState } from 'src/dashboard/types';
|
||||
import downloadAsPdf from 'src/utils/downloadAsPdf';
|
||||
import downloadAsImage from 'src/utils/downloadAsImage';
|
||||
import handleResourceExport from 'src/utils/export';
|
||||
import { redirect } from 'src/utils/navigationUtils';
|
||||
import {
|
||||
LOG_ACTIONS_DASHBOARD_DOWNLOAD_AS_PDF,
|
||||
LOG_ACTIONS_DASHBOARD_DOWNLOAD_AS_IMAGE,
|
||||
@@ -43,6 +45,23 @@ import { useToasts } from 'src/components/MessageToasts/withToasts';
|
||||
import { MenuItemTooltip } from 'src/components/Chart/DisabledMenuItemTooltip';
|
||||
import { DownloadScreenshotFormat } from './types';
|
||||
|
||||
// A guest/embedded session has no email address to be notified at, so rather
|
||||
// than wait on that notification the frontend polls for completion instead;
|
||||
// the same polling also drives the auto-download for a regular session,
|
||||
// which arrives before its export email in practice.
|
||||
const EXPORT_STATUS_POLL_INTERVAL_MS = 3000;
|
||||
const EXPORT_STATUS_POLL_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
// An embedded guest has no email fallback: if the client stops polling, a
|
||||
// slow-but-successful export is orphaned with no way to retrieve it. Outlive
|
||||
// the server's hard task budget (11 minutes) instead of racing it.
|
||||
const EMBEDDED_EXPORT_STATUS_POLL_TIMEOUT_MS = 12 * 60 * 1000;
|
||||
|
||||
interface ExportStatusResponse {
|
||||
status?: 'pending' | 'ready' | 'error';
|
||||
download_url?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface UseDownloadMenuItemsProps {
|
||||
pdfMenuItemTitle: string;
|
||||
imageMenuItemTitle: string;
|
||||
@@ -70,8 +89,27 @@ export const useDownloadMenuItems = (
|
||||
canExportImage,
|
||||
} = props;
|
||||
|
||||
const { addDangerToast, addSuccessToast } = useToasts();
|
||||
const { addDangerToast, addSuccessToast, addInfoToast } = useToasts();
|
||||
const dataMask = useSelector((state: RootState) => state.dataMask);
|
||||
// Embedded (iframe) sessions may have no email address, so they get
|
||||
// delivery-neutral copy and a poll window that outlives the task budget.
|
||||
const isEmbedded = isEmbeddedDashboard();
|
||||
const pollTimeoutMs = isEmbedded
|
||||
? EMBEDDED_EXPORT_STATUS_POLL_TIMEOUT_MS
|
||||
: EXPORT_STATUS_POLL_TIMEOUT_MS;
|
||||
|
||||
// Mirror the screenshot download's repeating info toast: re-shown on every
|
||||
// pending poll with noDuplicate, so the reminder persists for the export's
|
||||
// whole lifetime without stacking.
|
||||
const addExportPendingToast = () =>
|
||||
addInfoToast(
|
||||
isEmbedded
|
||||
? t('Your export is being generated. Please, do not leave the page.')
|
||||
: t(
|
||||
"Your export is being generated and will download automatically when ready. We'll also email you a download link.",
|
||||
),
|
||||
{ noDuplicate: true },
|
||||
);
|
||||
const SCREENSHOT_NODE_SELECTOR = '.dashboard';
|
||||
|
||||
const buildActiveDataMask = (): Record<string, { extraFormData: object }> =>
|
||||
@@ -167,6 +205,56 @@ export const useDownloadMenuItems = (
|
||||
}
|
||||
};
|
||||
|
||||
const pollExportStatus = (jobId: string, startedAt: number) => {
|
||||
SupersetClient.get({
|
||||
endpoint: `/api/v1/dashboard/export_xlsx/status/${jobId}/`,
|
||||
})
|
||||
.then(({ json }) => {
|
||||
const {
|
||||
status,
|
||||
download_url: downloadUrl,
|
||||
message,
|
||||
} = json as ExportStatusResponse;
|
||||
if (status === 'ready') {
|
||||
if (downloadUrl) {
|
||||
redirect(downloadUrl);
|
||||
}
|
||||
addSuccessToast(t('Your export is ready and downloading.'));
|
||||
return;
|
||||
}
|
||||
if (status === 'error') {
|
||||
addDangerToast(
|
||||
message || t('Sorry, something went wrong. Try again later.'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (Date.now() - startedAt > pollTimeoutMs) {
|
||||
addDangerToast(
|
||||
t('Your export is taking longer than expected. Try again later.'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
addExportPendingToast();
|
||||
setTimeout(
|
||||
() => pollExportStatus(jobId, startedAt),
|
||||
EXPORT_STATUS_POLL_INTERVAL_MS,
|
||||
);
|
||||
})
|
||||
.catch(error => {
|
||||
// A transient polling failure shouldn't give up the wait -- the export
|
||||
// itself may still succeed -- so keep polling until the timeout.
|
||||
logging.error(error);
|
||||
if (Date.now() - startedAt > pollTimeoutMs) {
|
||||
addDangerToast(t('Sorry, something went wrong. Try again later.'));
|
||||
return;
|
||||
}
|
||||
setTimeout(
|
||||
() => pollExportStatus(jobId, startedAt),
|
||||
EXPORT_STATUS_POLL_INTERVAL_MS,
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
const onExportXlsx = async (mode: 'data' | 'images') => {
|
||||
try {
|
||||
const { json } = await SupersetClient.post({
|
||||
@@ -175,11 +263,12 @@ export const useDownloadMenuItems = (
|
||||
});
|
||||
// The throttle response (an export is already running) returns 202 with a
|
||||
// message but no job_id; only a freshly enqueued job carries a job_id.
|
||||
if ((json as { job_id?: string })?.job_id) {
|
||||
addSuccessToast(
|
||||
t(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
),
|
||||
const jobId = (json as { job_id?: string })?.job_id;
|
||||
if (jobId) {
|
||||
addExportPendingToast();
|
||||
setTimeout(
|
||||
() => pollExportStatus(jobId, Date.now()),
|
||||
EXPORT_STATUS_POLL_INTERVAL_MS,
|
||||
);
|
||||
} else {
|
||||
addSuccessToast(
|
||||
@@ -255,8 +344,10 @@ export const useDownloadMenuItems = (
|
||||
// Image export renders charts through the headless webdriver, so only
|
||||
// offer it where that infrastructure is available (same signal as the
|
||||
// PDF/PNG image downloads above); otherwise non-table charts would
|
||||
// silently come back empty.
|
||||
...(isWebDriverScreenshotEnabled
|
||||
// silently come back empty. Embedded sessions are excluded too: the
|
||||
// webdriver cannot render Explore under a guest identity, so the
|
||||
// export would burn its whole task budget and produce nothing.
|
||||
...(isWebDriverScreenshotEnabled && !isEmbedded
|
||||
? [
|
||||
{
|
||||
key: 'export-xlsx-images',
|
||||
|
||||
+38
-13
@@ -66,6 +66,7 @@ from superset.tasks.types import ExecutorType
|
||||
from superset.themes.types import Theme
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.encrypt import SQLAlchemyUtilsAdapter
|
||||
from superset.utils.export_storage import ExportStorage
|
||||
from superset.utils.log import DBEventLogger
|
||||
from superset.utils.logging_configurator import DefaultLoggingConfigurator
|
||||
from superset.utils.version import get_dev_env_label
|
||||
@@ -368,6 +369,7 @@ WTF_CSRF_ENABLED = True
|
||||
WTF_CSRF_EXEMPT_LIST = [
|
||||
"superset.charts.data.api.data",
|
||||
"superset.dashboards.api.cache_dashboard_screenshot",
|
||||
"superset.dashboards.api.export_xlsx",
|
||||
"superset.views.core.log",
|
||||
"superset.views.datasource.views.samples",
|
||||
"flask_appbuilder.security.views.acs",
|
||||
@@ -1521,22 +1523,45 @@ CSV_STREAMING_ROW_THRESHOLD = 100000
|
||||
# note: index option should not be overridden
|
||||
EXCEL_EXPORT: dict[str, Any] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Dashboard "Export Data to Excel" (async, S3-backed)
|
||||
# Dashboard "Export Data to Excel" (async, object-storage-backed)
|
||||
# ---------------------------------------------------
|
||||
# Destination S3 bucket for generated dashboard .xlsx exports. The feature is
|
||||
# disabled until this is set: the export endpoint returns 501 when it is None.
|
||||
EXCEL_EXPORT_S3_BUCKET: str | None = None
|
||||
# Key prefix for export objects: {prefix}{dashboard_id}/{job_id}.xlsx
|
||||
EXCEL_EXPORT_S3_KEY_PREFIX = "dashboard-exports/"
|
||||
# Lifetime (seconds) of the pre-signed download URL emailed to the user (24h).
|
||||
# Note: AWS S3 caps pre-signed URL lifetime at 7 days (604800 seconds); larger
|
||||
# values are rejected by S3, so keep this at or below that when using AWS.
|
||||
class ExcelExportStorageConfig(TypedDict, total=False):
|
||||
"""Where dashboard Excel exports are uploaded, and how the download
|
||||
redirect resolves them back to a fresh URL. See EXCEL_EXPORT_STORAGE."""
|
||||
|
||||
# Destination bucket for generated dashboard .xlsx exports. The feature is
|
||||
# disabled until this is set: the export endpoint returns 501 while absent.
|
||||
bucket: str
|
||||
# Key/blob prefix for export objects: {prefix}{dashboard_id}/{job_id}.xlsx
|
||||
key_prefix: str
|
||||
# Extra kwargs passed to boto3.client("s3", ...) when using the default S3
|
||||
# backend (i.e. "backend" below is unset) — e.g. region_name, or an
|
||||
# endpoint_url for S3-compatible stores (MinIO/LocalStack). Credentials
|
||||
# otherwise resolve through the standard boto3 chain. Ignored by any
|
||||
# configured "backend", which authenticates however it authenticates.
|
||||
client_kwargs: dict[str, Any]
|
||||
# Optional pluggable storage backend (an instance implementing
|
||||
# superset.utils.export_storage.ExportStorage), the same pattern as
|
||||
# RESULTS_BACKEND or CUSTOM_SECURITY_MANAGER. When unset, the built-in
|
||||
# superset.utils.s3 (boto3/AWS S3) backend is used, configured by
|
||||
# "client_kwargs" above. Set this to e.g. GCSExportStorage()
|
||||
# (superset.utils.gcs) for a deployment whose "bucket" above names a
|
||||
# native Google Cloud Storage bucket rather than an S3 one.
|
||||
backend: ExportStorage
|
||||
|
||||
|
||||
EXCEL_EXPORT_STORAGE: ExcelExportStorageConfig = {
|
||||
"key_prefix": "dashboard-exports/",
|
||||
}
|
||||
# Lifetime (seconds) of the download link emailed to the user (24h). Not part
|
||||
# of ExcelExportStorageConfig: it bounds the Superset-issued redirect link
|
||||
# itself (see superset.dashboards.excel_export.download_link), independent of
|
||||
# how long the underlying storage backend's own credentials or URLs last.
|
||||
# Note: AWS S3 caps pre-signed URL lifetime at 7 days (604800 seconds), so
|
||||
# keep this at or below that when using the default S3 backend.
|
||||
EXCEL_EXPORT_LINK_TTL_SECONDS = 86400
|
||||
# Extra kwargs passed to boto3.client("s3", ...) — e.g. region_name, or an
|
||||
# endpoint_url for S3-compatible stores (MinIO/LocalStack). Credentials
|
||||
# otherwise resolve through the standard boto3 chain.
|
||||
EXCEL_EXPORT_S3_CLIENT_KWARGS: dict[str, Any] = {}
|
||||
# Viz types treated as tables in the "Export Images to Excel" mode: these charts
|
||||
# stay tabular (one worksheet of data) while every other viz type is embedded as
|
||||
# a rendered image. Set to None to fall back to the built-in default.
|
||||
|
||||
+134
-13
@@ -91,6 +91,14 @@ from superset.commands.importers.v1.utils import get_contents_from_bundle
|
||||
from superset.commands.purge import PurgeArchivedCommand, SoftDeleteBinding
|
||||
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod
|
||||
from superset.daos.dashboard import DashboardDAO, EmbeddedDashboardDAO
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
build_download_url,
|
||||
get_export_status,
|
||||
PRESIGNED_URL_TTL_SECONDS,
|
||||
resolve_download_link,
|
||||
STATUS_ERROR,
|
||||
STATUS_READY,
|
||||
)
|
||||
from superset.dashboards.filters import (
|
||||
DashboardAccessFilter,
|
||||
DashboardCertifiedFilter,
|
||||
@@ -155,8 +163,12 @@ from superset.tasks.thumbnails import (
|
||||
cache_dashboard_thumbnail,
|
||||
)
|
||||
from superset.tasks.utils import get_current_user
|
||||
from superset.utils import json
|
||||
from superset.utils.core import parse_boolean_string, sanitize_cookie_token
|
||||
from superset.utils import json, s3
|
||||
from superset.utils.core import (
|
||||
get_user_id,
|
||||
parse_boolean_string,
|
||||
sanitize_cookie_token,
|
||||
)
|
||||
from superset.utils.file import get_filename
|
||||
from superset.utils.pdf import build_pdf_from_screenshots
|
||||
from superset.utils.screenshots import (
|
||||
@@ -324,6 +336,8 @@ class DashboardRestApi(
|
||||
"put_colors",
|
||||
"export_as_example",
|
||||
"export_xlsx",
|
||||
"export_xlsx_status",
|
||||
"download_xlsx",
|
||||
"list_versions",
|
||||
"get_version",
|
||||
"activity",
|
||||
@@ -348,6 +362,9 @@ class DashboardRestApi(
|
||||
# menu item on it) instead of the ``can_export_xlsx`` FAB would otherwise
|
||||
# derive from the method name.
|
||||
"export_xlsx": "export",
|
||||
# Polling status of an export you already requested is the same
|
||||
# capability as requesting it, not a distinct permission.
|
||||
"export_xlsx_status": "export",
|
||||
"purge": "write",
|
||||
}
|
||||
|
||||
@@ -1723,8 +1740,11 @@ class DashboardRestApi(
|
||||
summary: Export dashboard chart data to Excel
|
||||
description: >-
|
||||
Enqueues an async task that writes each chart's data to its own
|
||||
worksheet, uploads the .xlsx to S3, and emails the requesting user a
|
||||
pre-signed download link. Returns immediately with a job id.
|
||||
worksheet, uploads the .xlsx to S3, and records a download link.
|
||||
The requesting user is emailed the link when they have an address
|
||||
on file; either way the returned job id can be polled at
|
||||
export_xlsx/status/<job_id>/ for status and, once ready, the
|
||||
download link.
|
||||
parameters:
|
||||
- in: path
|
||||
schema:
|
||||
@@ -1756,7 +1776,7 @@ class DashboardRestApi(
|
||||
501:
|
||||
description: Excel export is not configured on this server
|
||||
"""
|
||||
if not current_app.config["EXCEL_EXPORT_S3_BUCKET"]:
|
||||
if not current_app.config["EXCEL_EXPORT_STORAGE"].get("bucket"):
|
||||
return self.response(
|
||||
501, message="Excel export is not configured on this server."
|
||||
)
|
||||
@@ -1787,12 +1807,9 @@ class DashboardRestApi(
|
||||
except SupersetSecurityException:
|
||||
return self.response_403()
|
||||
|
||||
# Email delivery is the only result channel, so an account with an email
|
||||
# address is required; embedded guest users are excluded in this version.
|
||||
if isinstance(g.user, GuestUser) or not getattr(g.user, "email", None):
|
||||
return self.response_400(
|
||||
message="Excel export requires an account with an email address."
|
||||
)
|
||||
# A requester with no email on file (e.g. an embedded/guest session)
|
||||
# still gets a usable export: they poll export_xlsx_status/<job_id>/
|
||||
# for the download link instead of relying on an email notification.
|
||||
if not dashboard.slices:
|
||||
return self.response_400(message="Dashboard has no charts to export.")
|
||||
|
||||
@@ -1801,7 +1818,12 @@ class DashboardRestApi(
|
||||
# otherwise) so the guard works across the web server and workers and is
|
||||
# not a no-op under the default cache. The task releases it when it
|
||||
# settles; the TTL is the backstop if that release is ever lost.
|
||||
lock_params = export_lock_params(g.user.id, dashboard.id)
|
||||
# A guest/embedded requester has no DB-backed user id (GuestUser carries
|
||||
# no ``id`` attribute at all), so all guests share lock slot 0 for the
|
||||
# dashboard; the task reconstructs the guest (with the token's RLS rules
|
||||
# and resource claims) from the token payload passed alongside.
|
||||
user_id = get_user_id()
|
||||
lock_params = export_lock_params(user_id or 0, dashboard.id)
|
||||
try:
|
||||
AcquireDistributedLock(
|
||||
EXPORT_LOCK_NAMESPACE,
|
||||
@@ -1819,10 +1841,15 @@ class DashboardRestApi(
|
||||
export_dashboard_excel.apply_async(
|
||||
kwargs={
|
||||
"dashboard_id": dashboard.id,
|
||||
"user_id": g.user.id,
|
||||
"user_id": user_id,
|
||||
"active_data_mask": payload.get("active_data_mask", {}),
|
||||
"job_id": job_id,
|
||||
"mode": payload.get("mode", "data"),
|
||||
"guest_token": (
|
||||
getattr(g.user, "guest_token", None)
|
||||
if user_id is None
|
||||
else None
|
||||
),
|
||||
},
|
||||
task_id=job_id,
|
||||
)
|
||||
@@ -1834,6 +1861,100 @@ class DashboardRestApi(
|
||||
raise
|
||||
return self.response(202, job_id=job_id)
|
||||
|
||||
@expose("/export_xlsx/status/<uuid:job_id>/", methods=("GET",))
|
||||
@protect()
|
||||
@safe
|
||||
@statsd_metrics
|
||||
def export_xlsx_status(self, job_id: uuid.UUID) -> WerkzeugResponse:
|
||||
"""Poll the status of an in-flight or completed Excel export.
|
||||
---
|
||||
get:
|
||||
summary: Poll the status of a dashboard Excel export job
|
||||
description: >-
|
||||
For a session with no email address to be notified at (e.g. an
|
||||
embedded/guest session), the frontend polls this endpoint with the
|
||||
job_id from the export_xlsx response instead of waiting for an
|
||||
email. Behind the same @protect() as the export request itself,
|
||||
unlike the login-free download_xlsx redirect (which also has to
|
||||
work when clicked from a plain email link, possibly with no
|
||||
active session at all).
|
||||
parameters:
|
||||
- in: path
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
name: job_id
|
||||
description: The job_id from the export_xlsx response
|
||||
responses:
|
||||
200:
|
||||
description: >-
|
||||
Job status: {"status": "pending"} while still running,
|
||||
{"status": "ready", "download_url": "..."} once the file is
|
||||
available, or {"status": "error", "message": "..."} if the
|
||||
export failed.
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
"""
|
||||
payload = get_export_status(job_id)
|
||||
if payload is None:
|
||||
return self.response(200, status="pending")
|
||||
if payload.get("status") == STATUS_READY:
|
||||
return self.response(
|
||||
200, status=STATUS_READY, download_url=build_download_url(job_id)
|
||||
)
|
||||
if payload.get("status") == STATUS_ERROR:
|
||||
return self.response(
|
||||
200, status=STATUS_ERROR, message=payload.get("message")
|
||||
)
|
||||
return self.response(200, status="pending")
|
||||
|
||||
@expose("/export_xlsx/download/<uuid:job_id>/", methods=("GET",))
|
||||
@safe
|
||||
@statsd_metrics
|
||||
def download_xlsx(self, job_id: uuid.UUID) -> WerkzeugResponse:
|
||||
"""Redirect to a freshly pre-signed S3 URL for a completed Excel export.
|
||||
---
|
||||
get:
|
||||
summary: Download a completed dashboard Excel export
|
||||
description: >-
|
||||
Intentionally requires no login, matching a raw pre-signed S3
|
||||
URL's own access model: the unguessable job_id, emailed only to
|
||||
the original requester (or handed to their own session via
|
||||
export_xlsx_status), is the credential. The dashboard access
|
||||
check already ran once, when the export was requested -- see
|
||||
security_manager.raise_for_access in export_xlsx. A fresh
|
||||
pre-signed URL is generated at click time (instead of the one
|
||||
baked into the export at completion time) so the link's promised
|
||||
lifetime is independent of how long the signing credentials
|
||||
themselves remain valid.
|
||||
parameters:
|
||||
- in: path
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
name: job_id
|
||||
description: The job_id from the export_xlsx response
|
||||
responses:
|
||||
302:
|
||||
description: Redirect to a pre-signed S3 download URL
|
||||
410:
|
||||
description: The link is unknown, expired, or the export failed
|
||||
"""
|
||||
resolved = resolve_download_link(job_id)
|
||||
if resolved is None:
|
||||
return self.response(410, message="This download link has expired.")
|
||||
bucket, key = resolved
|
||||
storage_backend = current_app.config["EXCEL_EXPORT_STORAGE"].get("backend")
|
||||
if storage_backend is not None:
|
||||
download_url = storage_backend.generate_download_url(
|
||||
bucket, key, PRESIGNED_URL_TTL_SECONDS
|
||||
)
|
||||
else:
|
||||
download_url = s3.generate_presigned_url(
|
||||
bucket, key, PRESIGNED_URL_TTL_SECONDS
|
||||
)
|
||||
return redirect(download_url)
|
||||
|
||||
@expose("/<pk>/cache_dashboard_screenshot/", methods=("POST",))
|
||||
@validate_feature_flags(["THUMBNAILS", "ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS"])
|
||||
@protect()
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Status tracking and long-lived download links for dashboard Excel exports.
|
||||
|
||||
A raw S3 pre-signed URL is only valid for as long as *both* its own
|
||||
``ExpiresIn`` window and the credentials that signed it remain valid.
|
||||
Deployments whose S3 client authenticates via short-lived, auto-refreshed
|
||||
credentials (e.g. an EKS IRSA role assumed through
|
||||
``AssumeRoleWithWebIdentity``, which AWS caps at 12 hours and many clusters
|
||||
default to far less) can silently invalidate a pre-signed URL long before the
|
||||
``EXCEL_EXPORT_LINK_TTL_SECONDS`` window promised in the export email elapses,
|
||||
since the *credentials'* session -- not just the URL's own ``ExpiresIn`` --
|
||||
bounds how long it actually works.
|
||||
|
||||
To keep that promise regardless of credential lifetime, the email links to a
|
||||
small Superset redirect endpoint instead of a raw S3 URL. The link's own
|
||||
lifetime is enforced by this module via the ``key_value`` store's
|
||||
``expires_on`` (independent of any credential session), and the actual
|
||||
pre-signed URL is generated fresh -- with then-current credentials -- at click
|
||||
time, valid only long enough to complete a single download.
|
||||
|
||||
The redirect endpoint (``download_xlsx``) intentionally requires no login: a
|
||||
pre-signed S3 URL never did either, and the access-control decision for the
|
||||
underlying dashboard was already enforced once, when the export was
|
||||
originally requested (see ``security_manager.raise_for_access`` in
|
||||
``superset.dashboards.api.export_xlsx``). The unguessable key emailed only to
|
||||
that requester's own address is the same "possession of the link is the
|
||||
credential" model the raw pre-signed URL had; this module just re-signs it
|
||||
closer to when it is actually used.
|
||||
|
||||
Every entry is keyed by ``job_id`` -- the same id the ``export_xlsx`` POST
|
||||
response hands back -- rather than a separately-generated identifier, so a
|
||||
caller that only has the job id (e.g. a polling frontend for a session with
|
||||
no email on file, such as an embedded/guest dashboard) can resolve both
|
||||
status and, once ready, a download link from that one id.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from superset.daos.key_value import KeyValueDAO
|
||||
from superset.key_value.types import JsonKeyValueCodec, KeyValueResource
|
||||
from superset.utils.urls import headless_url
|
||||
|
||||
RESOURCE = KeyValueResource.EXCEL_EXPORT_DOWNLOAD
|
||||
CODEC = JsonKeyValueCodec()
|
||||
|
||||
# The fresh pre-signed URL generated at click time only needs to outlive the
|
||||
# redirect and the browser/S3 handshake that follows it, not the link's own
|
||||
# multi-hour lifetime.
|
||||
PRESIGNED_URL_TTL_SECONDS = 300
|
||||
|
||||
DOWNLOAD_PATH = "/api/v1/dashboard/export_xlsx/download/{job_id}/"
|
||||
|
||||
STATUS_READY = "ready"
|
||||
STATUS_ERROR = "error"
|
||||
|
||||
|
||||
def _sweep_and_upsert(
|
||||
job_id: UUID, value: dict[str, Any], expires_at: datetime
|
||||
) -> None:
|
||||
# Lazily sweep expired entries each time one is written; there is no
|
||||
# dedicated cleanup job, so this resource keeps itself tidy on write.
|
||||
# upsert (not create) so a retried/duplicate write for the same job_id
|
||||
# overwrites cleanly instead of colliding on the primary key.
|
||||
KeyValueDAO.delete_expired_entries(RESOURCE)
|
||||
KeyValueDAO.upsert_entry(
|
||||
resource=RESOURCE,
|
||||
value=value,
|
||||
codec=CODEC,
|
||||
key=job_id,
|
||||
expires_on=expires_at,
|
||||
)
|
||||
|
||||
|
||||
def build_download_url(job_id: UUID) -> str:
|
||||
"""The browser-facing URL that redirects to a freshly pre-signed S3 URL
|
||||
for ``job_id``, once its export is ready."""
|
||||
return headless_url(DOWNLOAD_PATH.format(job_id=job_id), user_friendly=True)
|
||||
|
||||
|
||||
def create_download_link(
|
||||
job_id: UUID, bucket: str, key: str, expires_at: datetime
|
||||
) -> str:
|
||||
"""Record that ``job_id``'s export succeeded and is downloadable from
|
||||
``key`` in ``bucket`` until ``expires_at``, and return the download URL
|
||||
(used in the success email).
|
||||
|
||||
``expires_at`` should be a naive datetime in the same timezone convention
|
||||
``KeyValueEntry.is_expired()`` compares against (naive ``datetime.now()``).
|
||||
"""
|
||||
_sweep_and_upsert(
|
||||
job_id,
|
||||
{"status": STATUS_READY, "bucket": bucket, "key": key},
|
||||
expires_at,
|
||||
)
|
||||
return build_download_url(job_id)
|
||||
|
||||
|
||||
def mark_export_failed(job_id: UUID, message: str, expires_at: datetime) -> None:
|
||||
"""Record that ``job_id``'s export failed, so a polling client can
|
||||
distinguish "failed" from "still running" instead of retrying a missing
|
||||
key forever. ``message`` is shown to whoever is polling, so keep it
|
||||
generic rather than an internal exception string.
|
||||
"""
|
||||
_sweep_and_upsert(job_id, {"status": STATUS_ERROR, "message": message}, expires_at)
|
||||
|
||||
|
||||
def get_export_status(job_id: UUID) -> dict[str, Any] | None:
|
||||
"""The stored status payload for ``job_id``, or ``None`` if it is unknown
|
||||
(still running, or never existed) or has expired."""
|
||||
return KeyValueDAO.get_value(RESOURCE, job_id, CODEC)
|
||||
|
||||
|
||||
def resolve_download_link(job_id: UUID) -> tuple[str, str] | None:
|
||||
"""The ``(bucket, object_key)`` for a *ready* download, or ``None`` if it
|
||||
is missing, expired, still running, or errored."""
|
||||
payload = get_export_status(job_id)
|
||||
if payload is None or payload.get("status") != STATUS_READY:
|
||||
return None
|
||||
return payload["bucket"], payload["key"]
|
||||
@@ -42,6 +42,7 @@ class KeyValueFilter(TypedDict, total=False):
|
||||
class KeyValueResource(StrEnum):
|
||||
APP = "app"
|
||||
DASHBOARD_PERMALINK = "dashboard_permalink"
|
||||
EXCEL_EXPORT_DOWNLOAD = "excel_export_download"
|
||||
EXPLORE_PERMALINK = "explore_permalink"
|
||||
METASTORE_CACHE = "superset_metastore_cache"
|
||||
LOCK = "lock"
|
||||
|
||||
@@ -16,8 +16,10 @@
|
||||
# under the License.
|
||||
"""
|
||||
Celery task that exports every chart on a dashboard to a single multi-sheet
|
||||
``.xlsx`` file, uploads it to S3, and emails the requesting user a pre-signed
|
||||
download link.
|
||||
``.xlsx`` file, uploads it to S3, and records a download link (see
|
||||
``superset.dashboards.excel_export.download_link``) that emails to the
|
||||
requesting user when they have an address on file, and/or is resolved by
|
||||
polling ``GET .../export_xlsx/status/<job_id>/`` when they don't.
|
||||
|
||||
In ``"data"`` mode the task re-runs each chart's saved query context under the
|
||||
requesting user, applies the live dashboard filter state, and streams the results
|
||||
@@ -33,6 +35,7 @@ import copy
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -53,9 +56,14 @@ from superset.common.form_data_query_context import (
|
||||
is_raw_query_mode,
|
||||
)
|
||||
from superset.dashboards.excel_export import email
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
create_download_link,
|
||||
mark_export_failed,
|
||||
)
|
||||
from superset.dashboards.excel_export.layout import get_charts_in_layout_order
|
||||
from superset.dashboards.excel_export.screenshot import render_chart_image
|
||||
from superset.extensions import celery_app
|
||||
from superset.security.guest_token import GuestToken
|
||||
from superset.utils import json, s3
|
||||
from superset.utils.core import override_user
|
||||
from superset.utils.excel_streaming import StreamingXlsxWriter
|
||||
@@ -299,6 +307,13 @@ def _write_chart_sheets(
|
||||
json_body["result_type"] = ChartDataResultType.FULL
|
||||
json_body.pop("force", None)
|
||||
|
||||
# Guest authorization links a chart to its dashboard through
|
||||
# ``form_data.dashboardId`` (raise_for_access); saved contexts don't carry
|
||||
# it, so stamp it the way the browser does on interactive requests.
|
||||
form_data = dict(json_body.get("form_data") or {})
|
||||
form_data["dashboardId"] = dashboard_id
|
||||
json_body["form_data"] = form_data
|
||||
|
||||
filter_context = get_dashboard_filter_context(
|
||||
dashboard_id=dashboard_id,
|
||||
chart_id=chart.id,
|
||||
@@ -400,19 +415,49 @@ def _build_workbook(
|
||||
return errored
|
||||
|
||||
|
||||
def _send_failure_email(
|
||||
user: Any, dashboard_title: str, requested_at: datetime
|
||||
_GENERIC_FAILURE_MESSAGE = (
|
||||
"An error occurred while generating the file. Please try again, or "
|
||||
"contact your administrator if the problem persists."
|
||||
)
|
||||
|
||||
|
||||
def _handle_export_failure(
|
||||
user: Any, dashboard_title: str, requested_at: datetime, job_id: str, ttl: int
|
||||
) -> None:
|
||||
if not (user and getattr(user, "email", None)):
|
||||
return
|
||||
"""Notify the requester their export failed: email them if they have an
|
||||
address on file, and record a pollable failure status either way (a
|
||||
session with no email, e.g. an embedded/guest dashboard, has no other way
|
||||
to learn the export failed than polling ``export_xlsx/status/<job_id>/``).
|
||||
"""
|
||||
if user and getattr(user, "email", None):
|
||||
try:
|
||||
email.send_export_email(
|
||||
user.email,
|
||||
email.build_subject(dashboard_title, success=False),
|
||||
email.build_failure_email(dashboard_title, requested_at),
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception("Failed to send export failure email")
|
||||
try:
|
||||
email.send_export_email(
|
||||
user.email,
|
||||
email.build_subject(dashboard_title, success=False),
|
||||
email.build_failure_email(dashboard_title, requested_at),
|
||||
expires_at = datetime.now(tz=timezone.utc) + timedelta(seconds=ttl)
|
||||
mark_export_failed(
|
||||
uuid.UUID(job_id),
|
||||
_GENERIC_FAILURE_MESSAGE,
|
||||
expires_at.replace(tzinfo=None),
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception("Failed to send export failure email")
|
||||
logger.exception("Failed to record export failure status for %s", job_id)
|
||||
|
||||
|
||||
def _upload_export_file(tmp_path: str, bucket: str, key: str) -> None:
|
||||
"""Upload the generated workbook via EXCEL_EXPORT_STORAGE's "backend" if
|
||||
configured (e.g. a GCS backend for a deployment whose bucket isn't S3),
|
||||
else the built-in boto3/S3 helper."""
|
||||
storage_backend = current_app.config["EXCEL_EXPORT_STORAGE"].get("backend")
|
||||
if storage_backend is not None:
|
||||
storage_backend.upload_file(tmp_path, bucket, key)
|
||||
else:
|
||||
s3.upload_file_to_s3(tmp_path, bucket, key)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
@@ -425,30 +470,45 @@ def _send_failure_email(
|
||||
def export_dashboard_excel(
|
||||
self: Any, # pylint: disable=unused-argument
|
||||
dashboard_id: int,
|
||||
user_id: int,
|
||||
user_id: int | None,
|
||||
active_data_mask: dict[str, Any],
|
||||
job_id: str,
|
||||
mode: str = EXPORT_MODE_DATA,
|
||||
guest_token: GuestToken | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Export a dashboard's charts to an ``.xlsx`` and email a download link.
|
||||
Export a dashboard's charts to an ``.xlsx`` and record a download link.
|
||||
|
||||
:param dashboard_id: The dashboard to export
|
||||
:param user_id: The requesting user (the task runs with their permissions)
|
||||
:param user_id: The requesting user (the task runs with their permissions),
|
||||
or ``None`` for a guest/embedded requester
|
||||
:param active_data_mask: Live dashboard filter state keyed by native filter id
|
||||
:param job_id: Correlation id, also the Celery task id and S3 object name
|
||||
:param mode: ``"data"`` streams every chart's tabular result; ``"images"``
|
||||
embeds non-table charts as rendered images and keeps tables tabular
|
||||
:param guest_token: The guest token payload when the requester is an
|
||||
embedded guest; the guest user is reconstructed from it so the export
|
||||
runs under the token's RLS rules and resource claims, never under an
|
||||
elevated identity
|
||||
"""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
requested_at = datetime.now(tz=timezone.utc)
|
||||
user = security_manager.get_user_by_id(user_id)
|
||||
user = None
|
||||
dashboard_title = ""
|
||||
tmp_path: str | None = None
|
||||
ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"]
|
||||
|
||||
try:
|
||||
# Resolve the user inside the protected block: if this raises (e.g. the
|
||||
# guest role lookup fails), the ``finally`` below must still release the
|
||||
# lock the API acquired, and the failure status must still be recorded
|
||||
# for pollers.
|
||||
if user_id is not None:
|
||||
user = security_manager.get_user_by_id(user_id)
|
||||
elif guest_token:
|
||||
user = security_manager.get_guest_user_from_token(guest_token)
|
||||
with override_user(user, force=False):
|
||||
dashboard = (
|
||||
db.session.query(Dashboard).filter_by(id=dashboard_id).one_or_none()
|
||||
@@ -466,16 +526,18 @@ def export_dashboard_excel(
|
||||
tmp_path, dashboard, active_data_mask, job_id, mode, user
|
||||
)
|
||||
|
||||
bucket = current_app.config["EXCEL_EXPORT_S3_BUCKET"]
|
||||
key = (
|
||||
f"{current_app.config['EXCEL_EXPORT_S3_KEY_PREFIX']}"
|
||||
f"{dashboard_id}/{job_id}.xlsx"
|
||||
)
|
||||
ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"]
|
||||
storage_config = current_app.config["EXCEL_EXPORT_STORAGE"]
|
||||
bucket = storage_config.get("bucket")
|
||||
key_prefix = storage_config.get("key_prefix", "dashboard-exports/")
|
||||
key = f"{key_prefix}{dashboard_id}/{job_id}.xlsx"
|
||||
|
||||
s3.upload_file_to_s3(tmp_path, bucket, key)
|
||||
download_url = s3.generate_presigned_url(bucket, key, ttl)
|
||||
_upload_export_file(tmp_path, bucket, key)
|
||||
expires_at = datetime.now(tz=timezone.utc) + timedelta(seconds=ttl)
|
||||
# KeyValueEntry.expires_on comparisons use naive datetime.now(), so
|
||||
# the stored expiry must be naive UTC too, not tz-aware.
|
||||
download_url = create_download_link(
|
||||
uuid.UUID(job_id), bucket, key, expires_at.replace(tzinfo=None)
|
||||
)
|
||||
|
||||
if user and getattr(user, "email", None):
|
||||
try:
|
||||
@@ -497,17 +559,18 @@ def export_dashboard_excel(
|
||||
logger.exception("Failed to send export success email")
|
||||
except SoftTimeLimitExceeded:
|
||||
logger.warning("Dashboard excel export %s timed out", job_id)
|
||||
_send_failure_email(user, dashboard_title, requested_at)
|
||||
_handle_export_failure(user, dashboard_title, requested_at, job_id, ttl)
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Dashboard excel export %s failed", job_id)
|
||||
_send_failure_email(user, dashboard_title, requested_at)
|
||||
_handle_export_failure(user, dashboard_title, requested_at, job_id, ttl)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
ReleaseDistributedLock(
|
||||
EXPORT_LOCK_NAMESPACE,
|
||||
export_lock_params(user_id, dashboard_id),
|
||||
# Must mirror the key the API acquired: guests share slot 0.
|
||||
export_lock_params(user_id or 0, dashboard_id),
|
||||
).run()
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# Best-effort: the lock's TTL is the backstop if this fails.
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Pluggable storage backend interface for dashboard Excel export artifacts.
|
||||
|
||||
Setting ``EXCEL_EXPORT_STORAGE["backend"]`` in ``superset_config.py`` to an
|
||||
instance of a class implementing this protocol (the same "instance in
|
||||
config" pattern as ``RESULTS_BACKEND`` or ``CUSTOM_SECURITY_MANAGER``) swaps
|
||||
out where the export task uploads the generated ``.xlsx`` and how the
|
||||
download redirect mints a fresh, time-limited URL for it. When unset,
|
||||
``superset.utils.s3`` (boto3/AWS S3) is used.
|
||||
|
||||
This module has no dependency on any storage SDK: it is safe to import (e.g.
|
||||
from ``superset/config.py``) regardless of which storage extras, if any, are
|
||||
installed. A concrete implementation -- such as a hypothetical
|
||||
``GCSExportStorage`` for deployments where the export bucket is a native
|
||||
Google Cloud Storage bucket rather than S3 -- imports its own SDK lazily, the
|
||||
same way ``superset.utils.s3`` only imports ``boto3`` inside the functions
|
||||
that need it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ExportStorage(Protocol):
|
||||
"""Where the export task uploads a file, and how a download link resolves
|
||||
it back to a fresh, time-limited URL at click time.
|
||||
|
||||
The two operations run at very different times against the same object:
|
||||
``upload_file`` once, when the export finishes; ``generate_download_url``
|
||||
every time the (possibly long-lived) download link is clicked, so its
|
||||
credentials never need to outlive the link itself. See
|
||||
``superset.dashboards.excel_export.download_link`` for why the link is a
|
||||
Superset redirect rather than a raw storage URL.
|
||||
"""
|
||||
|
||||
def upload_file(self, local_path: str, bucket: str, key: str) -> None:
|
||||
"""Upload a local file to ``bucket``/``key``."""
|
||||
|
||||
def generate_download_url(self, bucket: str, key: str, expires_in: int) -> str:
|
||||
"""A time-limited URL for downloading ``bucket``/``key``, valid for
|
||||
``expires_in`` seconds from now."""
|
||||
@@ -0,0 +1,85 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
``ExportStorage`` implementation backed by Google Cloud Storage, for
|
||||
deployments where the dashboard Excel export bucket is a native GCS bucket
|
||||
rather than S3.
|
||||
|
||||
Set ``EXCEL_EXPORT_STORAGE["backend"] = GCSExportStorage()`` in
|
||||
``superset_config.py`` to use this instead of the default ``superset.utils.s3``
|
||||
(boto3/AWS) backend. Authentication uses Application Default Credentials (a
|
||||
service account key, workload identity, etc.) via the standard
|
||||
``google-cloud-storage`` resolution chain -- there is no separate credential
|
||||
config here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _get_client() -> Any:
|
||||
"""Build a GCS client using Application Default Credentials."""
|
||||
# Imported lazily, mirroring superset.utils.s3._get_s3_client: importing
|
||||
# this module (which happens at config-load time if EXCEL_EXPORT_STORAGE
|
||||
# is set) should not require google-cloud-storage unless an export
|
||||
# actually runs.
|
||||
try:
|
||||
from google.cloud import storage # pylint: disable=import-outside-toplevel
|
||||
except ImportError as ex:
|
||||
raise ImportError(
|
||||
"google-cloud-storage is required for GCSExportStorage but is not "
|
||||
"installed. Install it with "
|
||||
"`pip install apache-superset[excel-export-gcs]`."
|
||||
) from ex
|
||||
|
||||
return storage.Client()
|
||||
|
||||
|
||||
class GCSExportStorage:
|
||||
"""``ExportStorage`` backed by Google Cloud Storage.
|
||||
|
||||
See ``superset.utils.export_storage.ExportStorage`` for the interface this
|
||||
implements.
|
||||
"""
|
||||
|
||||
def upload_file(self, local_path: str, bucket: str, key: str) -> None:
|
||||
"""
|
||||
Upload a local file to GCS.
|
||||
|
||||
:param local_path: Path to the file on local disk
|
||||
:param bucket: Destination GCS bucket
|
||||
:param key: Destination GCS blob name
|
||||
"""
|
||||
_get_client().bucket(bucket).blob(key).upload_from_filename(local_path)
|
||||
|
||||
def generate_download_url(self, bucket: str, key: str, expires_in: int) -> str:
|
||||
"""
|
||||
Generate a time-limited signed URL for downloading a GCS object.
|
||||
|
||||
:param bucket: The GCS bucket
|
||||
:param key: The GCS blob name
|
||||
:param expires_in: URL lifetime in seconds
|
||||
:returns: A v4 signed URL
|
||||
"""
|
||||
blob = _get_client().bucket(bucket).blob(key)
|
||||
return blob.generate_signed_url(
|
||||
version="v4",
|
||||
expiration=timedelta(seconds=expires_in),
|
||||
method="GET",
|
||||
)
|
||||
@@ -19,7 +19,7 @@ Minimal S3 helpers for uploading export artifacts and minting pre-signed URLs.
|
||||
|
||||
Credentials and region come from the standard boto3 resolution chain (env vars,
|
||||
shared config, instance role). Operators can override client construction via
|
||||
the ``EXCEL_EXPORT_S3_CLIENT_KWARGS`` config (e.g. ``region_name`` or an
|
||||
``EXCEL_EXPORT_STORAGE["client_kwargs"]`` (e.g. ``region_name`` or an
|
||||
``endpoint_url`` for S3-compatible stores such as MinIO/LocalStack).
|
||||
"""
|
||||
|
||||
@@ -47,8 +47,8 @@ def _get_s3_client() -> Any:
|
||||
"Install it with `pip install apache-superset[excel-export]`."
|
||||
) from ex
|
||||
|
||||
client_kwargs: dict[str, Any] = current_app.config.get(
|
||||
"EXCEL_EXPORT_S3_CLIENT_KWARGS", {}
|
||||
client_kwargs: dict[str, Any] = current_app.config["EXCEL_EXPORT_STORAGE"].get(
|
||||
"client_kwargs", {}
|
||||
)
|
||||
return boto3.client("s3", **client_kwargs)
|
||||
|
||||
|
||||
@@ -17,11 +17,15 @@
|
||||
# isort:skip_file
|
||||
"""Unit tests for Superset"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
from unittest.mock import ANY, patch
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
from zipfile import is_zipfile, ZipFile
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from tests.integration_tests.insert_chart_mixin import InsertChartMixin
|
||||
|
||||
import pytest
|
||||
@@ -3408,7 +3412,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_404_for_missing_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 404 for an unknown dashboard."""
|
||||
@@ -3417,7 +3421,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
assert rv.status_code == 404
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_400_for_empty_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 400 for a dashboard with no charts."""
|
||||
@@ -3433,7 +3437,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_202_enqueues_task(self, mock_task, mock_acquire):
|
||||
@@ -3456,7 +3460,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
assert kwargs["kwargs"]["dashboard_id"] == dashboard.id
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_202_when_export_already_in_progress(
|
||||
@@ -3475,7 +3479,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
assert "already in progress" in rv.data.decode("utf-8")
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_404_for_inaccessible_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 404 for a dashboard the user can't see."""
|
||||
@@ -3493,7 +3497,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
@patch("superset.dashboards.api.security_manager.raise_for_access")
|
||||
@@ -3524,7 +3528,184 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_admitted_without_email(self, mock_task, mock_acquire):
|
||||
"""Dashboard API: a session with no email address (what an
|
||||
embedded/guest session looks like from this check's perspective) is
|
||||
admitted (202), not rejected -- the requesting user no longer needs an
|
||||
email on file, since the frontend can poll
|
||||
export_xlsx_status/<job_id>/ for the download link instead of relying
|
||||
on a notification email."""
|
||||
admin_user = security_manager.find_user(username=ADMIN_USERNAME)
|
||||
slice_ = db.session.query(Slice).first()
|
||||
# Clone Admin (so the login password is valid), then blank the email
|
||||
# to match what an embedded/guest session looks like to this check.
|
||||
with self.temporary_user(admin_user, login=True) as user:
|
||||
user.email = ""
|
||||
db.session.commit()
|
||||
dashboard = self.insert_dashboard(
|
||||
"xlsx-no-email", None, [user.id], slices=[slice_], published=True
|
||||
)
|
||||
try:
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}},
|
||||
)
|
||||
assert rv.status_code == 202
|
||||
mock_task.apply_async.assert_called_once()
|
||||
finally:
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
def test_download_xlsx_redirects_without_login(self):
|
||||
"""Dashboard API: download_xlsx requires no login, matching a raw
|
||||
pre-signed S3 URL's own access model -- the dashboard access check
|
||||
already ran once, when the export was requested."""
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
create_download_link,
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
create_download_link(
|
||||
job_id,
|
||||
"exports",
|
||||
"dashboard-exports/1/job.xlsx",
|
||||
datetime.now() + timedelta(hours=1),
|
||||
)
|
||||
db.session.commit()
|
||||
with patch("superset.dashboards.api.s3.generate_presigned_url") as mock_sign:
|
||||
mock_sign.return_value = "https://bucket.s3.amazonaws.com/signed"
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/download/{job_id}/")
|
||||
assert rv.status_code == 302
|
||||
assert rv.headers["Location"] == "https://bucket.s3.amazonaws.com/signed"
|
||||
mock_sign.assert_called_once_with(
|
||||
"exports", "dashboard-exports/1/job.xlsx", ANY
|
||||
)
|
||||
|
||||
def test_download_xlsx_uses_configured_storage_backend(self):
|
||||
"""Dashboard API: EXCEL_EXPORT_STORAGE (e.g. GCSExportStorage()) takes
|
||||
over minting the download URL entirely; the default boto3/S3 helper
|
||||
must not also run."""
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
create_download_link,
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
create_download_link(
|
||||
job_id,
|
||||
"exports",
|
||||
"dashboard-exports/1/job.xlsx",
|
||||
datetime.now() + timedelta(hours=1),
|
||||
)
|
||||
db.session.commit()
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.generate_download_url.return_value = (
|
||||
"https://storage.googleapis.com/signed"
|
||||
)
|
||||
original_storage_config = current_app.config["EXCEL_EXPORT_STORAGE"]
|
||||
current_app.config["EXCEL_EXPORT_STORAGE"] = {"backend": mock_storage}
|
||||
try:
|
||||
with patch(
|
||||
"superset.dashboards.api.s3.generate_presigned_url"
|
||||
) as mock_sign:
|
||||
rv = self.client.get(
|
||||
f"/api/v1/dashboard/export_xlsx/download/{job_id}/"
|
||||
)
|
||||
finally:
|
||||
current_app.config["EXCEL_EXPORT_STORAGE"] = original_storage_config
|
||||
assert rv.status_code == 302
|
||||
assert rv.headers["Location"] == "https://storage.googleapis.com/signed"
|
||||
mock_storage.generate_download_url.assert_called_once_with(
|
||||
"exports", "dashboard-exports/1/job.xlsx", ANY
|
||||
)
|
||||
mock_sign.assert_not_called()
|
||||
|
||||
def test_download_xlsx_410_for_unknown_key(self):
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/download/{uuid.uuid4()}/")
|
||||
assert rv.status_code == 410
|
||||
|
||||
def test_download_xlsx_410_for_expired_key(self):
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
create_download_link,
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
create_download_link(
|
||||
job_id,
|
||||
"exports",
|
||||
"dashboard-exports/1/job.xlsx",
|
||||
datetime.now() - timedelta(hours=1),
|
||||
)
|
||||
db.session.commit()
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/download/{job_id}/")
|
||||
assert rv.status_code == 410
|
||||
|
||||
def test_download_xlsx_410_for_errored_job(self):
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
mark_export_failed,
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
mark_export_failed(job_id, "boom", datetime.now() + timedelta(hours=1))
|
||||
db.session.commit()
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/download/{job_id}/")
|
||||
assert rv.status_code == 410
|
||||
|
||||
def test_export_xlsx_status_pending_for_unknown_job(self):
|
||||
"""Dashboard API: polling an unknown/still-running job_id reports
|
||||
pending, not 404 -- the frontend can't distinguish "not started yet"
|
||||
from "still running" from the API's perspective."""
|
||||
self.login(ADMIN_USERNAME)
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/status/{uuid.uuid4()}/")
|
||||
assert rv.status_code == 200
|
||||
assert rv.json == {"status": "pending"}
|
||||
|
||||
@patch("superset.dashboards.api.s3.generate_presigned_url")
|
||||
def test_export_xlsx_status_ready_includes_download_url(self, mock_presign):
|
||||
"""Dashboard API: once ready, status includes a download_url built
|
||||
from the same job_id, not a separately-tracked identifier."""
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
create_download_link,
|
||||
)
|
||||
|
||||
mock_presign.return_value = "https://bucket.s3.amazonaws.com/signed"
|
||||
job_id = uuid.uuid4()
|
||||
create_download_link(
|
||||
job_id,
|
||||
"exports",
|
||||
"dashboard-exports/1/job.xlsx",
|
||||
datetime.now() + timedelta(hours=1),
|
||||
)
|
||||
db.session.commit()
|
||||
self.login(ADMIN_USERNAME)
|
||||
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/status/{job_id}/")
|
||||
|
||||
assert rv.status_code == 200
|
||||
assert rv.json["status"] == "ready"
|
||||
assert str(job_id) in rv.json["download_url"]
|
||||
|
||||
def test_export_xlsx_status_error_includes_message(self):
|
||||
"""Dashboard API: a failed job's status is distinguishable from
|
||||
pending, with a message a polling guest session can show."""
|
||||
from superset.dashboards.excel_export.download_link import (
|
||||
mark_export_failed,
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
mark_export_failed(job_id, "boom", datetime.now() + timedelta(hours=1))
|
||||
db.session.commit()
|
||||
self.login(ADMIN_USERNAME)
|
||||
|
||||
rv = self.client.get(f"/api/v1/dashboard/export_xlsx/status/{job_id}/")
|
||||
|
||||
assert rv.status_code == 200
|
||||
assert rv.json == {"status": "error", "message": "boom"}
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_images_404_when_screenshot_flags_off(self, mock_task):
|
||||
"""Dashboard API: ``mode=images`` is rejected with 404 when the webdriver
|
||||
@@ -3540,7 +3721,7 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_config({"EXCEL_EXPORT_STORAGE": {"bucket": "exports"}})
|
||||
@with_feature_flags(
|
||||
ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS=True,
|
||||
ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT=True,
|
||||
|
||||
@@ -1750,6 +1750,14 @@ class TestRolePermission(SupersetTestCase):
|
||||
# user/tenant data) as content-addressed scripts; must load for
|
||||
# anonymous principals (login page, embedded dashboards).
|
||||
["Superset", "language_pack_script"],
|
||||
# Intentionally unauthenticated, matching a raw pre-signed S3 URL's
|
||||
# own access model: the unguessable job_id in the path is the
|
||||
# credential, and the underlying dashboard access check already
|
||||
# ran once, when the export was requested (see
|
||||
# security_manager.raise_for_access in export_xlsx). Must be
|
||||
# reachable with no session at all, since it is clicked from a
|
||||
# plain email link.
|
||||
["DashboardRestApi", "download_xlsx"],
|
||||
]
|
||||
unsecured_views = []
|
||||
for view_class in appbuilder.baseviews:
|
||||
|
||||
@@ -27,10 +27,19 @@ from unittest import mock
|
||||
import pytest
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
|
||||
from superset.security.guest_token import GuestToken
|
||||
from superset.utils import json
|
||||
|
||||
MODULE = "superset.tasks.export_dashboard_excel"
|
||||
|
||||
# export_dashboard_excel always receives a real uuid4 job_id in production (the
|
||||
# API generates it); use valid UUIDs here too since the task parses job_id via
|
||||
# uuid.UUID() to key the download-link/status store.
|
||||
JOB_ID = "00000000-0000-0000-0000-000000000001"
|
||||
JOB_ID_TIMEOUT = "00000000-0000-0000-0000-000000000002"
|
||||
JOB_ID_IMG_TIMEOUT = "00000000-0000-0000-0000-000000000003"
|
||||
JOB_ID_FAIL = "00000000-0000-0000-0000-000000000004"
|
||||
|
||||
|
||||
# A minimal valid 1x1 transparent PNG for image-mode tests.
|
||||
_PNG_1x1 = (
|
||||
@@ -84,6 +93,8 @@ def mocks() -> Iterator[dict[str, Any]]:
|
||||
"s3",
|
||||
"email",
|
||||
"ReleaseDistributedLock",
|
||||
"create_download_link",
|
||||
"mark_export_failed",
|
||||
)
|
||||
}
|
||||
user = mock.MagicMock()
|
||||
@@ -100,7 +111,7 @@ def mocks() -> Iterator[dict[str, Any]]:
|
||||
)
|
||||
|
||||
patched["get_dashboard_filter_context"].return_value.extra_form_data = {}
|
||||
patched["s3"].generate_presigned_url.return_value = "https://signed/file.xlsx"
|
||||
patched["create_download_link"].return_value = "https://signed/file.xlsx"
|
||||
|
||||
patched["user"] = user
|
||||
patched["dashboard"] = dashboard
|
||||
@@ -108,7 +119,7 @@ def mocks() -> Iterator[dict[str, Any]]:
|
||||
|
||||
|
||||
def _run(
|
||||
job_id: str = "job-1",
|
||||
job_id: str = JOB_ID,
|
||||
mode: str = "data",
|
||||
) -> None:
|
||||
from superset.tasks.export_dashboard_excel import export_dashboard_excel
|
||||
@@ -162,7 +173,28 @@ def test_happy_path_uploads_and_emails(mocks: dict[str, Any]) -> None:
|
||||
assert list(uploaded["sheets"].keys()) == ["10 - First", "20 - Second"]
|
||||
mocks["email"].send_export_email.assert_called_once()
|
||||
mocks["email"].build_success_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-1")
|
||||
assert _no_temp_files_left(JOB_ID)
|
||||
|
||||
|
||||
def test_upload_uses_configured_storage_backend(mocks: dict[str, Any]) -> None:
|
||||
# EXCEL_EXPORT_STORAGE (e.g. GCSExportStorage()) takes over the upload
|
||||
# entirely; the default boto3/S3 helper must not also run.
|
||||
from flask import current_app
|
||||
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
mock_storage = mock.MagicMock()
|
||||
original_storage_config = current_app.config["EXCEL_EXPORT_STORAGE"]
|
||||
current_app.config["EXCEL_EXPORT_STORAGE"] = {"backend": mock_storage}
|
||||
try:
|
||||
_run()
|
||||
finally:
|
||||
current_app.config["EXCEL_EXPORT_STORAGE"] = original_storage_config
|
||||
|
||||
mock_storage.upload_file.assert_called_once()
|
||||
mocks["s3"].upload_file_to_s3.assert_not_called()
|
||||
|
||||
|
||||
def test_chart_without_query_context_is_skipped(mocks: dict[str, Any]) -> None:
|
||||
@@ -296,8 +328,10 @@ def _builder_hook(builder: Any) -> Iterator[None]:
|
||||
# Real values for the keys the task subscripts directly, so a full export can
|
||||
# run under the hook (a MagicMock ttl would blow up building the link expiry).
|
||||
fake_app.config.__getitem__.side_effect = {
|
||||
"EXCEL_EXPORT_S3_BUCKET": "bucket",
|
||||
"EXCEL_EXPORT_S3_KEY_PREFIX": "dashboard-exports/",
|
||||
"EXCEL_EXPORT_STORAGE": {
|
||||
"bucket": "bucket",
|
||||
"key_prefix": "dashboard-exports/",
|
||||
},
|
||||
"EXCEL_EXPORT_LINK_TTL_SECONDS": 3600,
|
||||
}.__getitem__
|
||||
with mock.patch.object(module, "current_app", fake_app):
|
||||
@@ -658,12 +692,12 @@ def test_chart_timeout_aborts_export_and_sends_failure_email(
|
||||
]
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-timeout")
|
||||
_run(JOB_ID_TIMEOUT)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.assert_not_called()
|
||||
mocks["email"].build_success_email.assert_not_called()
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-timeout")
|
||||
assert _no_temp_files_left(JOB_ID_TIMEOUT)
|
||||
|
||||
|
||||
def test_image_render_timeout_aborts_export(mocks: dict[str, Any]) -> None:
|
||||
@@ -675,11 +709,11 @@ def test_image_render_timeout_aborts_export(mocks: dict[str, Any]) -> None:
|
||||
mocks["render_chart_image"].side_effect = SoftTimeLimitExceeded()
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-img-timeout", mode="images")
|
||||
_run(JOB_ID_IMG_TIMEOUT, mode="images")
|
||||
|
||||
mocks["email"].build_success_email.assert_not_called()
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-img-timeout")
|
||||
assert _no_temp_files_left(JOB_ID_IMG_TIMEOUT)
|
||||
|
||||
|
||||
def test_all_charts_skipped_writes_summary(mocks: dict[str, Any]) -> None:
|
||||
@@ -709,21 +743,21 @@ def test_upload_failure_sends_failure_email_and_cleans_up(
|
||||
mocks["s3"].upload_file_to_s3.side_effect = RuntimeError("s3 down")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
_run("job-fail")
|
||||
_run(JOB_ID_FAIL)
|
||||
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
mocks["email"].send_export_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-fail")
|
||||
assert _no_temp_files_left(JOB_ID_FAIL)
|
||||
|
||||
|
||||
def test_soft_time_limit_sends_failure_email(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].side_effect = SoftTimeLimitExceeded()
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-timeout")
|
||||
_run(JOB_ID_TIMEOUT)
|
||||
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-timeout")
|
||||
assert _no_temp_files_left(JOB_ID_TIMEOUT)
|
||||
|
||||
|
||||
# --- image mode ---
|
||||
@@ -808,6 +842,24 @@ def test_images_mode_none_render_is_skipped(mocks: dict[str, Any]) -> None:
|
||||
assert "Export Summary" in uploaded["sheets"]
|
||||
|
||||
|
||||
def test_query_context_is_stamped_with_the_dashboard_id(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
"""Guest datasource authorization links a chart to its dashboard through
|
||||
form_data.dashboardId (raise_for_access); the browser stamps it on every
|
||||
interactive request and the task must do the same when replaying a saved
|
||||
context, or every chart in a guest export fails the access check."""
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
(payload,), _ = mocks["ChartDataQueryContextSchema"].return_value.load.call_args
|
||||
assert payload["form_data"]["dashboardId"] == 1
|
||||
|
||||
|
||||
def test_inflight_lock_released_on_success(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
@@ -824,6 +876,77 @@ def test_inflight_lock_released_on_success(mocks: dict[str, Any]) -> None:
|
||||
mocks["ReleaseDistributedLock"].return_value.run.assert_called_once_with()
|
||||
|
||||
|
||||
def test_guest_export_reconstructs_guest_user_and_shares_lock_slot_zero(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
"""A guest export (user_id=None) rebuilds the user from the token payload —
|
||||
so the token's RLS rules apply in the worker — and releases lock slot 0,
|
||||
mirroring the key the API acquired for guests."""
|
||||
from superset.tasks.export_dashboard_excel import export_dashboard_excel
|
||||
|
||||
# Like a real GuestUser: no ``email`` (and no ``id``) attribute at all.
|
||||
guest = mock.MagicMock(spec=["username"])
|
||||
mocks["security_manager"].get_guest_user_from_token.return_value = guest
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
token: GuestToken = {
|
||||
"iat": 0.0,
|
||||
"exp": 0.0,
|
||||
"user": {},
|
||||
"resources": [],
|
||||
"rls_rules": [],
|
||||
}
|
||||
|
||||
export_dashboard_excel(
|
||||
dashboard_id=1,
|
||||
user_id=None,
|
||||
active_data_mask={},
|
||||
job_id=JOB_ID,
|
||||
guest_token=token,
|
||||
)
|
||||
|
||||
mocks["security_manager"].get_guest_user_from_token.assert_called_once_with(token)
|
||||
mocks["security_manager"].get_user_by_id.assert_not_called()
|
||||
# Guests have no email address, so no notification is attempted.
|
||||
mocks["email"].send_export_email.assert_not_called()
|
||||
# The file still lands in S3 for the status-poll download path.
|
||||
mocks["s3"].upload_file_to_s3.assert_called_once()
|
||||
mocks["ReleaseDistributedLock"].assert_called_once_with(
|
||||
"excel_export", {"user_id": 0, "dashboard_id": 1}
|
||||
)
|
||||
|
||||
|
||||
def test_lock_released_and_failure_recorded_when_user_resolution_fails(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
"""If reconstructing the requester fails (e.g. the guest role lookup
|
||||
raises), the lock the API acquired is still released and the failure is
|
||||
still recorded for pollers."""
|
||||
from superset.tasks.export_dashboard_excel import export_dashboard_excel
|
||||
|
||||
mocks["security_manager"].get_guest_user_from_token.side_effect = RuntimeError(
|
||||
"role lookup failed"
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
export_dashboard_excel(
|
||||
dashboard_id=1,
|
||||
user_id=None,
|
||||
active_data_mask={},
|
||||
job_id=JOB_ID_FAIL,
|
||||
guest_token=GuestToken(
|
||||
iat=0.0, exp=0.0, user={}, resources=[], rls_rules=[]
|
||||
),
|
||||
)
|
||||
|
||||
mocks["ReleaseDistributedLock"].assert_called_once_with(
|
||||
"excel_export", {"user_id": 0, "dashboard_id": 1}
|
||||
)
|
||||
mocks["mark_export_failed"].assert_called_once()
|
||||
|
||||
|
||||
def test_inflight_lock_released_on_failure(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
@@ -832,7 +955,7 @@ def test_inflight_lock_released_on_failure(mocks: dict[str, Any]) -> None:
|
||||
mocks["s3"].upload_file_to_s3.side_effect = RuntimeError("s3 down")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
_run("job-fail")
|
||||
_run(JOB_ID_FAIL)
|
||||
|
||||
# The lock is freed in ``finally`` even when the export fails.
|
||||
mocks["ReleaseDistributedLock"].assert_called_once_with(
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils.export_storage import ExportStorage
|
||||
from superset.utils.gcs import GCSExportStorage
|
||||
|
||||
|
||||
@patch("superset.utils.gcs._get_client")
|
||||
def test_upload_file(mock_get_client: MagicMock) -> None:
|
||||
blob = mock_get_client.return_value.bucket.return_value.blob.return_value
|
||||
|
||||
GCSExportStorage().upload_file(
|
||||
"exports/out.xlsx", "my-bucket", "exports/1/abc.xlsx"
|
||||
)
|
||||
|
||||
mock_get_client.return_value.bucket.assert_called_once_with("my-bucket")
|
||||
mock_get_client.return_value.bucket.return_value.blob.assert_called_once_with(
|
||||
"exports/1/abc.xlsx"
|
||||
)
|
||||
blob.upload_from_filename.assert_called_once_with("exports/out.xlsx")
|
||||
|
||||
|
||||
@patch("superset.utils.gcs._get_client")
|
||||
def test_generate_download_url(mock_get_client: MagicMock) -> None:
|
||||
blob = mock_get_client.return_value.bucket.return_value.blob.return_value
|
||||
blob.generate_signed_url.return_value = "https://storage.googleapis.com/signed"
|
||||
|
||||
url = GCSExportStorage().generate_download_url(
|
||||
"my-bucket", "exports/1/abc.xlsx", 300
|
||||
)
|
||||
|
||||
assert url == "https://storage.googleapis.com/signed"
|
||||
blob.generate_signed_url.assert_called_once_with(
|
||||
version="v4",
|
||||
expiration=timedelta(seconds=300),
|
||||
method="GET",
|
||||
)
|
||||
|
||||
|
||||
def test_implements_export_storage_protocol() -> None:
|
||||
assert isinstance(GCSExportStorage(), ExportStorage)
|
||||
|
||||
|
||||
def test_get_client_missing_dependency_raises_actionable_error() -> None:
|
||||
# Regression: without google-cloud-storage installed (it is only an
|
||||
# optional install), calling into GCSExportStorage must surface an
|
||||
# actionable hint rather than a bare ModuleNotFoundError. This is the
|
||||
# real behavior in this test environment, where the dependency isn't
|
||||
# installed -- no mocking needed to exercise it.
|
||||
from superset.utils import gcs
|
||||
|
||||
with pytest.raises(ImportError, match="excel-export-gcs"):
|
||||
gcs._get_client()
|
||||
@@ -27,7 +27,7 @@ from superset.utils import s3
|
||||
@patch("boto3.client")
|
||||
@patch("superset.utils.s3.current_app")
|
||||
def test_upload_file_to_s3(mock_app: MagicMock, mock_client_fn: MagicMock) -> None:
|
||||
mock_app.config = {"EXCEL_EXPORT_S3_CLIENT_KWARGS": {}}
|
||||
mock_app.config = {"EXCEL_EXPORT_STORAGE": {}}
|
||||
client = mock_client_fn.return_value
|
||||
|
||||
s3.upload_file_to_s3("exports/out.xlsx", "my-bucket", "exports/1/abc.xlsx")
|
||||
@@ -44,9 +44,11 @@ def test_client_kwargs_passthrough(
|
||||
mock_app: MagicMock, mock_client_fn: MagicMock
|
||||
) -> None:
|
||||
mock_app.config = {
|
||||
"EXCEL_EXPORT_S3_CLIENT_KWARGS": {
|
||||
"endpoint_url": "http://minio:9000",
|
||||
"region_name": "us-east-1",
|
||||
"EXCEL_EXPORT_STORAGE": {
|
||||
"client_kwargs": {
|
||||
"endpoint_url": "http://minio:9000",
|
||||
"region_name": "us-east-1",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +83,7 @@ def test_get_s3_client_missing_boto3_raises_actionable_error() -> None:
|
||||
@patch("boto3.client")
|
||||
@patch("superset.utils.s3.current_app")
|
||||
def test_generate_presigned_url(mock_app: MagicMock, mock_client_fn: MagicMock) -> None:
|
||||
mock_app.config = {"EXCEL_EXPORT_S3_CLIENT_KWARGS": {}}
|
||||
mock_app.config = {"EXCEL_EXPORT_STORAGE": {}}
|
||||
client = mock_client_fn.return_value
|
||||
client.generate_presigned_url.return_value = "https://signed.example/abc"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user