mirror of
https://github.com/apache/superset.git
synced 2026-07-18 04:35:40 +00:00
Compare commits
52 Commits
dependabot
...
hughhhh/da
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
46465d03fe | ||
|
|
7836890a9f | ||
|
|
54ef804de4 | ||
|
|
5ac547a3d8 | ||
|
|
11bfc3026e | ||
|
|
210dd228c5 | ||
|
|
2adff6a8d8 | ||
|
|
27496d4fbf | ||
|
|
a94835ed06 | ||
|
|
eac1480eb6 | ||
|
|
c8d61d7d1d | ||
|
|
6c0c8450e4 | ||
|
|
5616a867ce | ||
|
|
767db18b5e | ||
|
|
cbbf5382e8 | ||
|
|
108c34691e | ||
|
|
fe52c69a88 | ||
|
|
6f40ebc357 | ||
|
|
d4ac5fed84 | ||
|
|
5f7f6adb53 | ||
|
|
e3a7bf776e | ||
|
|
3602619e98 | ||
|
|
216cf65086 | ||
|
|
debf81db2f | ||
|
|
c812fc5d20 | ||
|
|
a05d53842c | ||
|
|
4c8fbce952 | ||
|
|
1f1b77b0ff | ||
|
|
e6a242bcfe | ||
|
|
b6b40457b9 | ||
|
|
aea0eac72c | ||
|
|
2f1666ff67 | ||
|
|
40a8804e7e | ||
|
|
2d7ea4cebb | ||
|
|
712e1de330 | ||
|
|
73e33cc60b | ||
|
|
667252a3fe | ||
|
|
b9e7dcfa77 | ||
|
|
a8c1148aad | ||
|
|
161c4d81d7 | ||
|
|
cc9d6f5d7d | ||
|
|
ba7454bba5 | ||
|
|
849f284af1 | ||
|
|
cea084898c | ||
|
|
7df440538a | ||
|
|
39412c0ac3 | ||
|
|
3730d22456 | ||
|
|
dbea4b6c87 | ||
|
|
5e708fd46a | ||
|
|
7668f4d75c | ||
|
|
5f5595e8df | ||
|
|
5421799b68 |
25
UPDATING.md
25
UPDATING.md
@@ -24,6 +24,31 @@ assists people when migrating to a new version.
|
||||
|
||||
## Next
|
||||
|
||||
### Dashboard "Export Data to Excel" requires a Celery worker and S3 bucket
|
||||
|
||||
A new dashboard action exports every chart's data to a single multi-sheet
|
||||
`.xlsx` asynchronously. It is disabled by default and turns on only when
|
||||
`EXCEL_EXPORT_S3_BUCKET` is set (the endpoint returns `501` otherwise). It also
|
||||
requires a running Celery worker and a configured SMTP transport, since the task
|
||||
emails the requesting user a pre-signed download link. New config keys:
|
||||
`EXCEL_EXPORT_S3_BUCKET`, `EXCEL_EXPORT_S3_KEY_PREFIX`,
|
||||
`EXCEL_EXPORT_LINK_TTL_SECONDS`, `EXCEL_EXPORT_S3_CLIENT_KWARGS`, and
|
||||
`EXCEL_EXPORT_TABLE_VIZ_TYPES`.
|
||||
|
||||
The feature depends on `boto3`, which is **not** installed by default; install it
|
||||
with `pip install apache-superset[excel-export]`.
|
||||
|
||||
A second mode, **Export Images to Excel**, embeds non-table charts as rendered
|
||||
images (which viz types stay tabular is controlled by
|
||||
`EXCEL_EXPORT_TABLE_VIZ_TYPES`). It renders through the headless webdriver, so the
|
||||
menu option only appears when the webdriver screenshot feature flags
|
||||
(`ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS`,
|
||||
`ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT`) are enabled.
|
||||
|
||||
Deployments that override `CELERY_CONFIG` must add
|
||||
`"superset.tasks.export_dashboard_excel"` to their `imports` tuple, or the task
|
||||
will not register and exports will silently never run.
|
||||
|
||||
### Owners, dashboard roles, and RLS roles replaced by Subjects
|
||||
|
||||
Superset now uses subject-based access assignments for dashboards, charts, datasets,
|
||||
|
||||
@@ -87,6 +87,7 @@ class CeleryConfig:
|
||||
"superset.tasks.scheduler",
|
||||
"superset.tasks.thumbnails",
|
||||
"superset.tasks.cache",
|
||||
"superset.tasks.export_dashboard_excel",
|
||||
)
|
||||
result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
|
||||
worker_prefetch_multiplier = 1
|
||||
|
||||
99
docs/docs/using-superset/exporting-dashboard-data.mdx
Normal file
99
docs/docs/using-superset/exporting-dashboard-data.mdx
Normal file
@@ -0,0 +1,99 @@
|
||||
---
|
||||
title: Exporting Dashboard Data to Excel
|
||||
hide_title: true
|
||||
sidebar_position: 7
|
||||
version: 1
|
||||
---
|
||||
|
||||
# Exporting Dashboard Data to Excel
|
||||
|
||||
Superset can export every chart on a dashboard to a single Excel workbook, with
|
||||
each chart's underlying data rendered as its own worksheet. The export reflects
|
||||
the dashboard's currently applied filters and runs asynchronously: when it
|
||||
finishes, the requesting user receives an email with a time-limited download
|
||||
link.
|
||||
|
||||
## Using the export
|
||||
|
||||
From a dashboard, open the **... (actions) → Download** submenu and choose
|
||||
**Export Data to Excel**. The action appears for users who have the dashboard
|
||||
`can_export` permission. You'll see a confirmation that the export is being
|
||||
prepared; the workbook arrives by email when it's ready.
|
||||
|
||||
A second option, **Export Images to Excel**, embeds each non-table chart as a
|
||||
rendered image (tables stay tabular) instead of exporting raw data. Because it
|
||||
renders charts through the headless webdriver, this option only appears when the
|
||||
webdriver screenshot feature flags are enabled (see the prerequisites below);
|
||||
which viz types stay tabular is controlled by `EXCEL_EXPORT_TABLE_VIZ_TYPES`.
|
||||
|
||||
Notes on the generated workbook:
|
||||
|
||||
- One worksheet per chart, named `{chart_id} - {chart title}` (truncated to
|
||||
Excel's 31-character limit; the chart id keeps names unique).
|
||||
- Charts nested in tabs are included.
|
||||
- Data reflects the dashboard's active filter state at the time of export.
|
||||
- A chart with no saved query context is skipped and listed in the email; open
|
||||
the chart in Explore and re-save it to include it next time.
|
||||
- Row counts per sheet are capped the same way as the chart-level CSV/Excel
|
||||
export (`ROW_LIMIT`, bounded by `SQL_MAX_ROW`), and never exceed Excel's
|
||||
per-sheet maximum.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
This feature is **disabled by default**. It requires:
|
||||
|
||||
1. **The `boto3` dependency.** It is not installed by default; install it with
|
||||
`pip install apache-superset[excel-export]`. Without it, exports fail and the
|
||||
user receives a failure email.
|
||||
2. **An S3 bucket.** Set `EXCEL_EXPORT_S3_BUCKET`. Until it is set, the export
|
||||
endpoint returns `501` and the menu action surfaces a "not configured"
|
||||
message.
|
||||
3. **A running Celery worker.** The export runs as a Celery task. If no worker
|
||||
is running, the request is accepted but nothing is produced.
|
||||
4. **A configured SMTP transport.** The download link is delivered by email
|
||||
using the same settings as alerts & reports (`SMTP_*`,
|
||||
`EMAIL_REPORTS_SUBJECT_PREFIX`).
|
||||
|
||||
**Export Images to Excel** additionally requires a working headless webdriver —
|
||||
the same infrastructure scheduled reports and thumbnails use (`WEBDRIVER_*`,
|
||||
plus the `ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS` and
|
||||
`ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT` feature flags). The menu option
|
||||
is hidden when those flags are off; if the webdriver is unreachable, image
|
||||
charts come back empty even though the data export path still works.
|
||||
|
||||
Deployments that override `CELERY_CONFIG` must add
|
||||
`"superset.tasks.export_dashboard_excel"` to the `imports` tuple, or the task
|
||||
will not register.
|
||||
|
||||
## Configuration keys
|
||||
|
||||
| Key | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `EXCEL_EXPORT_S3_BUCKET` | `None` | Destination bucket. Required; `501` if unset. |
|
||||
| `EXCEL_EXPORT_S3_KEY_PREFIX` | `"dashboard-exports/"` | Key prefix: `{prefix}{dashboard_id}/{job_id}.xlsx`. |
|
||||
| `EXCEL_EXPORT_LINK_TTL_SECONDS` | `86400` | Lifetime of the pre-signed download URL (24h). |
|
||||
| `EXCEL_EXPORT_S3_CLIENT_KWARGS` | `{}` | Extra kwargs for `boto3.client("s3", ...)` — e.g. `region_name`, or `endpoint_url` for MinIO/LocalStack. |
|
||||
| `EXCEL_EXPORT_TABLE_VIZ_TYPES` | `None` | Viz types kept tabular in **Export Images to Excel** mode; every other type is embedded as an image. `None` uses the built-in default (`table`, `pivot_table`, `pivot_table_v2`). |
|
||||
|
||||
Credentials and region resolve through the standard boto3 chain (environment
|
||||
variables, shared config, or instance role) unless overridden via
|
||||
`EXCEL_EXPORT_S3_CLIENT_KWARGS`. The worker needs `s3:PutObject` on the bucket.
|
||||
|
||||
## Security considerations
|
||||
|
||||
- The emailed link is a **pre-signed S3 URL**: anyone who holds it can download
|
||||
the workbook until it expires. Keep the bucket **private**, enable
|
||||
encryption, and consider a lifecycle rule to delete objects after a few days.
|
||||
Lower `EXCEL_EXPORT_LINK_TTL_SECONDS` if 24 hours is too long for your data.
|
||||
- The export runs with the requesting user's permissions; each chart's query is
|
||||
access-checked, so users only ever receive data they are entitled to.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Embedded dashboards / guest tokens are not supported** in this version,
|
||||
because guest users have no email address to deliver the link to. Logged-in
|
||||
users viewing an embedded dashboard can still use the export.
|
||||
- The default **Export Data to Excel** mode exports data only (no visual
|
||||
styling). Use **Export Images to Excel** to embed rendered chart images, which
|
||||
requires the webdriver infrastructure described in the prerequisites.
|
||||
- Scheduled/automated exports are not part of this feature.
|
||||
@@ -154,6 +154,10 @@ solr = ["sqlalchemy-solr >= 0.2.4.3"]
|
||||
elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"]
|
||||
exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"]
|
||||
excel = ["xlrd>=2.0.2, <2.1"]
|
||||
# Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and
|
||||
# emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so
|
||||
# installing this extra is only required to actually run exports.
|
||||
excel-export = ["boto3"]
|
||||
fastmcp = [
|
||||
"fastmcp>=3.4.3,<4.0",
|
||||
# tiktoken backs the response-size-guard token estimator. Without
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import { Menu, MenuItem } from '@superset-ui/core/components/Menu';
|
||||
import {
|
||||
FeatureFlag,
|
||||
getClientErrorObject,
|
||||
isFeatureEnabled,
|
||||
SupersetClient,
|
||||
} from '@superset-ui/core';
|
||||
@@ -46,12 +47,15 @@ jest.mock('src/components/MessageToasts/withToasts', () => ({
|
||||
jest.mock('@superset-ui/core', () => ({
|
||||
...jest.requireActual('@superset-ui/core'),
|
||||
isFeatureEnabled: jest.fn().mockReturnValue(false),
|
||||
getClientErrorObject: jest.fn().mockResolvedValue({}),
|
||||
SupersetClient: {
|
||||
get: jest.fn(),
|
||||
post: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const mockSupersetClient = SupersetClient as jest.Mocked<typeof SupersetClient>;
|
||||
const mockGetClientErrorObject = getClientErrorObject as jest.Mock;
|
||||
|
||||
const createProps = () => ({
|
||||
pdfMenuItemTitle: 'Export to PDF',
|
||||
@@ -70,19 +74,40 @@ const MenuWrapper = () => {
|
||||
return <Menu forceSubMenuRender items={menuItems} />;
|
||||
};
|
||||
|
||||
const MenuWrapperWithProps = (
|
||||
overrides: Partial<ReturnType<typeof createProps>> & {
|
||||
canExportImage?: boolean;
|
||||
},
|
||||
) => {
|
||||
const downloadMenuItem = useDownloadMenuItems({
|
||||
...createProps(),
|
||||
...overrides,
|
||||
});
|
||||
const menuItems: MenuItem[] = [downloadMenuItem];
|
||||
return <Menu forceSubMenuRender items={menuItems} />;
|
||||
};
|
||||
|
||||
const originalCreateObjectURL = window.URL.createObjectURL;
|
||||
const originalRevokeObjectURL = window.URL.revokeObjectURL;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
// Reset the implementation each test: clearAllMocks resets call history but
|
||||
// not mockReturnValue, so an override in one test would otherwise leak.
|
||||
(isFeatureEnabled as jest.Mock).mockReturnValue(false);
|
||||
});
|
||||
|
||||
// "Export Images to Excel" is gated on the webdriver screenshot feature flags.
|
||||
const enableWebDriverScreenshot = () =>
|
||||
(isFeatureEnabled as jest.Mock).mockReturnValue(true);
|
||||
|
||||
afterEach(() => {
|
||||
window.URL.createObjectURL = originalCreateObjectURL;
|
||||
window.URL.revokeObjectURL = originalRevokeObjectURL;
|
||||
});
|
||||
|
||||
test('Should render all menu items', () => {
|
||||
enableWebDriverScreenshot();
|
||||
render(<MenuWrapper />, {
|
||||
useRedux: true,
|
||||
});
|
||||
@@ -92,10 +117,120 @@ test('Should render all menu items', () => {
|
||||
expect(screen.getByText('Download as Image')).toBeInTheDocument();
|
||||
|
||||
// Export options
|
||||
expect(screen.getByText('Export Data to Excel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Export Images to Excel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Export YAML')).toBeInTheDocument();
|
||||
expect(screen.getByText('Export as Example')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('Export Images to Excel is hidden when the webdriver is not enabled', () => {
|
||||
// Default: webdriver screenshot flags off. Image export needs the webdriver,
|
||||
// so only the data export is offered.
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
expect(screen.getByText('Export Data to Excel')).toBeInTheDocument();
|
||||
expect(screen.queryByText('Export Images to Excel')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('Excel export items are hidden when userCanExport is false', () => {
|
||||
render(<MenuWrapperWithProps userCanExport={false} />, { useRedux: true });
|
||||
|
||||
expect(screen.queryByText('Export Data to Excel')).not.toBeInTheDocument();
|
||||
expect(screen.queryByText('Export Images to Excel')).not.toBeInTheDocument();
|
||||
// YAML export is not gated and remains visible
|
||||
expect(screen.getByText('Export YAML')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('Export Data to Excel posts mode "data" and shows a pending toast', async () => {
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockSupersetClient.post).toHaveBeenCalledWith({
|
||||
endpoint: '/api/v1/dashboard/123/export_xlsx/',
|
||||
jsonPayload: { active_data_mask: {}, mode: 'data' },
|
||||
});
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Images to Excel posts mode "images" and shows a pending toast', async () => {
|
||||
enableWebDriverScreenshot();
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: { job_id: 'abc' },
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Images to Excel'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockSupersetClient.post).toHaveBeenCalledWith({
|
||||
endpoint: '/api/v1/dashboard/123/export_xlsx/',
|
||||
jsonPayload: { active_data_mask: {}, mode: 'images' },
|
||||
});
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Data to Excel shows an "already in progress" toast when throttled', async () => {
|
||||
// The throttle response is 202 with a message but no job_id.
|
||||
mockSupersetClient.post.mockResolvedValue({
|
||||
json: {
|
||||
message: 'An Excel export for this dashboard is already in progress.',
|
||||
},
|
||||
} as never);
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockAddSuccessToast).toHaveBeenCalledWith(
|
||||
'An export for this dashboard is already in progress.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Data to Excel shows a config error toast on 501', async () => {
|
||||
mockSupersetClient.post.mockRejectedValue(new Error('not configured'));
|
||||
mockGetClientErrorObject.mockResolvedValue({ status: 501 });
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockAddDangerToast).toHaveBeenCalledWith(
|
||||
'Excel export is not configured on this server.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export Data to Excel shows a generic error toast on other failures', async () => {
|
||||
mockSupersetClient.post.mockRejectedValue(new Error('boom'));
|
||||
mockGetClientErrorObject.mockResolvedValue({ status: 500 });
|
||||
|
||||
render(<MenuWrapper />, { useRedux: true });
|
||||
|
||||
await userEvent.click(screen.getByText('Export Data to Excel'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockAddDangerToast).toHaveBeenCalledWith(
|
||||
'Sorry, something went wrong. Try again later.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('Export as Example calls SupersetClient.get with correct endpoint', async () => {
|
||||
const mockBlob = new Blob(['test'], { type: 'application/zip' });
|
||||
const mockResponse: Pick<Response, 'blob' | 'headers'> = {
|
||||
@@ -144,19 +279,6 @@ test('Export as Example shows error toast on failure', async () => {
|
||||
|
||||
const mockIsFeatureEnabled = isFeatureEnabled as jest.Mock;
|
||||
|
||||
const MenuWrapperWithProps = (
|
||||
overrides: Partial<ReturnType<typeof createProps>> & {
|
||||
canExportImage?: boolean;
|
||||
},
|
||||
) => {
|
||||
const downloadMenuItem = useDownloadMenuItems({
|
||||
...createProps(),
|
||||
...overrides,
|
||||
});
|
||||
const menuItems: MenuItem[] = [downloadMenuItem];
|
||||
return <Menu forceSubMenuRender items={menuItems} />;
|
||||
};
|
||||
|
||||
test('Screenshot menu items should be disabled when GranularExportControls is ON and canExportImage is false', () => {
|
||||
mockIsFeatureEnabled.mockImplementation(
|
||||
(flag: string) => flag === FeatureFlag.GranularExportControls,
|
||||
|
||||
@@ -17,17 +17,20 @@
|
||||
* under the License.
|
||||
*/
|
||||
import { SyntheticEvent } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
import { logging } from '@apache-superset/core/utils';
|
||||
import { t } from '@apache-superset/core/translation';
|
||||
import {
|
||||
FeatureFlag,
|
||||
getClientErrorObject,
|
||||
isFeatureEnabled,
|
||||
SupersetClient,
|
||||
} from '@superset-ui/core';
|
||||
import { MenuItem } from '@superset-ui/core/components/Menu';
|
||||
import { parse as parseContentDisposition } from 'content-disposition';
|
||||
import { useDownloadScreenshot } from 'src/dashboard/hooks/useDownloadScreenshot';
|
||||
import { MenuKeys } from 'src/dashboard/types';
|
||||
import { NATIVE_FILTER_PREFIX } from 'src/dashboard/components/nativeFilters/FiltersConfigModal/utils';
|
||||
import { MenuKeys, RootState } from 'src/dashboard/types';
|
||||
import downloadAsPdf from 'src/utils/downloadAsPdf';
|
||||
import downloadAsImage from 'src/utils/downloadAsImage';
|
||||
import handleResourceExport from 'src/utils/export';
|
||||
@@ -68,8 +71,19 @@ export const useDownloadMenuItems = (
|
||||
} = props;
|
||||
|
||||
const { addDangerToast, addSuccessToast } = useToasts();
|
||||
const dataMask = useSelector((state: RootState) => state.dataMask);
|
||||
const SCREENSHOT_NODE_SELECTOR = '.dashboard';
|
||||
|
||||
const buildActiveDataMask = (): Record<string, { extraFormData: object }> =>
|
||||
Object.entries(dataMask || {}).reduce<
|
||||
Record<string, { extraFormData: object }>
|
||||
>((acc, [id, mask]) => {
|
||||
if (id.startsWith(NATIVE_FILTER_PREFIX)) {
|
||||
acc[id] = { extraFormData: mask?.extraFormData ?? {} };
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const isWebDriverScreenshotEnabled =
|
||||
isFeatureEnabled(FeatureFlag.EnableDashboardScreenshotEndpoints) &&
|
||||
isFeatureEnabled(FeatureFlag.EnableDashboardDownloadWebDriverScreenshot);
|
||||
@@ -153,6 +167,39 @@ export const useDownloadMenuItems = (
|
||||
}
|
||||
};
|
||||
|
||||
const onExportXlsx = async (mode: 'data' | 'images') => {
|
||||
try {
|
||||
const { json } = await SupersetClient.post({
|
||||
endpoint: `/api/v1/dashboard/${dashboardId}/export_xlsx/`,
|
||||
jsonPayload: { active_data_mask: buildActiveDataMask(), mode },
|
||||
});
|
||||
// The throttle response (an export is already running) returns 202 with a
|
||||
// message but no job_id; only a freshly enqueued job carries a job_id.
|
||||
if ((json as { job_id?: string })?.job_id) {
|
||||
addSuccessToast(
|
||||
t(
|
||||
"Your export is being prepared. You'll receive an email when it's ready.",
|
||||
),
|
||||
);
|
||||
} else {
|
||||
addSuccessToast(
|
||||
t('An export for this dashboard is already in progress.'),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// status comes from the response (Partial<SupersetClientResponse>), which
|
||||
// the union type does not expose uniformly; read it via a narrow cast.
|
||||
const { status } = (await getClientErrorObject(error)) as {
|
||||
status?: number;
|
||||
};
|
||||
if (status === 501) {
|
||||
addDangerToast(t('Excel export is not configured on this server.'));
|
||||
} else {
|
||||
addDangerToast(t('Sorry, something went wrong. Try again later.'));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const imageDisabled = canExportImage === false;
|
||||
|
||||
const imageExportLabel = (text: string) =>
|
||||
@@ -198,6 +245,28 @@ export const useDownloadMenuItems = (
|
||||
];
|
||||
|
||||
const exportMenuItems: MenuItem[] = [
|
||||
...(userCanExport
|
||||
? [
|
||||
{
|
||||
key: 'export-xlsx',
|
||||
label: t('Export Data to Excel'),
|
||||
onClick: () => onExportXlsx('data'),
|
||||
},
|
||||
// Image export renders charts through the headless webdriver, so only
|
||||
// offer it where that infrastructure is available (same signal as the
|
||||
// PDF/PNG image downloads above); otherwise non-table charts would
|
||||
// silently come back empty.
|
||||
...(isWebDriverScreenshotEnabled
|
||||
? [
|
||||
{
|
||||
key: 'export-xlsx-images',
|
||||
label: t('Export Images to Excel'),
|
||||
onClick: () => onExportXlsx('images'),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]
|
||||
: []),
|
||||
{
|
||||
key: 'export-yaml',
|
||||
label: t('Export YAML'),
|
||||
|
||||
@@ -199,6 +199,31 @@ def _extract_filter_extra_form_data(
|
||||
return None, DashboardFilterStatus.NOT_APPLIED
|
||||
|
||||
|
||||
def _resolve_filter_extra_form_data(
|
||||
filter_config: dict[str, Any],
|
||||
active_data_mask: dict[str, Any] | None,
|
||||
) -> tuple[dict[str, Any] | None, DashboardFilterStatus]:
|
||||
"""
|
||||
Resolve a filter's extra_form_data and status, preferring an active value
|
||||
from ``active_data_mask`` over the filter's saved default.
|
||||
|
||||
When ``active_data_mask`` provides an entry for this filter, its
|
||||
``extraFormData`` is authoritative: a non-empty value is APPLIED, while an
|
||||
empty value means the user explicitly cleared the filter (NOT_APPLIED, with
|
||||
no fallback to the saved default). When no active entry exists, fall back to
|
||||
the saved-default behavior in ``_extract_filter_extra_form_data``.
|
||||
|
||||
Returns (extra_form_data, status).
|
||||
"""
|
||||
flt_id = filter_config.get("id", "")
|
||||
if active_data_mask is not None and flt_id in active_data_mask:
|
||||
active_efd = (active_data_mask[flt_id] or {}).get("extraFormData") or {}
|
||||
if active_efd:
|
||||
return active_efd, DashboardFilterStatus.APPLIED
|
||||
return None, DashboardFilterStatus.NOT_APPLIED
|
||||
return _extract_filter_extra_form_data(filter_config)
|
||||
|
||||
|
||||
def _get_filter_target_column(filter_config: dict[str, Any]) -> str | None:
|
||||
"""Extract the target column name from a native filter configuration."""
|
||||
if targets := filter_config.get("targets", []):
|
||||
@@ -244,16 +269,26 @@ def _check_dashboard_access(dashboard: Dashboard) -> None:
|
||||
def get_dashboard_filter_context(
|
||||
dashboard_id: int,
|
||||
chart_id: int,
|
||||
*,
|
||||
active_data_mask: dict[str, Any] | None = None,
|
||||
) -> DashboardFilterContext:
|
||||
"""
|
||||
Build a DashboardFilterContext for a chart on a dashboard.
|
||||
|
||||
Loads the dashboard's native filter configuration, determines which
|
||||
filters are in scope for the given chart, extracts default filter values,
|
||||
filters are in scope for the given chart, resolves each filter's value,
|
||||
and returns the merged extra_form_data along with metadata about each filter.
|
||||
|
||||
When ``active_data_mask`` is provided (e.g. the live filter state from a
|
||||
dashboard view), each in-scope filter present in the mask uses its active
|
||||
``extraFormData`` instead of the saved default; an empty active value means
|
||||
the filter was cleared. Filters absent from the mask fall back to their
|
||||
saved defaults, so omitting ``active_data_mask`` reproduces the dashboard's
|
||||
initial-load behavior.
|
||||
|
||||
:param dashboard_id: The ID of the dashboard
|
||||
:param chart_id: The ID of the chart
|
||||
:param active_data_mask: Optional live filter state keyed by native filter id
|
||||
:returns: DashboardFilterContext with merged extra_form_data and filter metadata
|
||||
:raises ValueError: if dashboard not found or chart not on dashboard
|
||||
:raises SupersetSecurityException: if the user cannot access the dashboard
|
||||
@@ -287,7 +322,7 @@ def get_dashboard_filter_context(
|
||||
flt_id = flt.get("id", "")
|
||||
flt_name = flt.get("name", "")
|
||||
target_column = _get_filter_target_column(flt)
|
||||
extra_form_data, status = _extract_filter_extra_form_data(flt)
|
||||
extra_form_data, status = _resolve_filter_extra_form_data(flt, active_data_mask)
|
||||
|
||||
if extra_form_data and status == DashboardFilterStatus.APPLIED:
|
||||
context.extra_form_data = _merge_extra_form_data(
|
||||
|
||||
@@ -1442,6 +1442,27 @@ CSV_STREAMING_ROW_THRESHOLD = 100000
|
||||
# note: index option should not be overridden
|
||||
EXCEL_EXPORT: dict[str, Any] = {}
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Dashboard "Export Data to Excel" (async, S3-backed)
|
||||
# ---------------------------------------------------
|
||||
# Destination S3 bucket for generated dashboard .xlsx exports. The feature is
|
||||
# disabled until this is set: the export endpoint returns 501 when it is None.
|
||||
EXCEL_EXPORT_S3_BUCKET: str | None = None
|
||||
# Key prefix for export objects: {prefix}{dashboard_id}/{job_id}.xlsx
|
||||
EXCEL_EXPORT_S3_KEY_PREFIX = "dashboard-exports/"
|
||||
# Lifetime (seconds) of the pre-signed download URL emailed to the user (24h).
|
||||
# Note: AWS S3 caps pre-signed URL lifetime at 7 days (604800 seconds); larger
|
||||
# values are rejected by S3, so keep this at or below that when using AWS.
|
||||
EXCEL_EXPORT_LINK_TTL_SECONDS = 86400
|
||||
# Extra kwargs passed to boto3.client("s3", ...) — e.g. region_name, or an
|
||||
# endpoint_url for S3-compatible stores (MinIO/LocalStack). Credentials
|
||||
# otherwise resolve through the standard boto3 chain.
|
||||
EXCEL_EXPORT_S3_CLIENT_KWARGS: dict[str, Any] = {}
|
||||
# Viz types treated as tables in the "Export Images to Excel" mode: these charts
|
||||
# stay tabular (one worksheet of data) while every other viz type is embedded as
|
||||
# a rendered image. Set to None to fall back to the built-in default.
|
||||
EXCEL_EXPORT_TABLE_VIZ_TYPES: set[str] | None = None
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Time grain configurations
|
||||
# ---------------------------------------------------
|
||||
@@ -1634,6 +1655,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods
|
||||
"superset.tasks.thumbnails",
|
||||
"superset.tasks.cache",
|
||||
"superset.tasks.slack",
|
||||
"superset.tasks.export_dashboard_excel",
|
||||
)
|
||||
result_backend = "db+sqlite:///celery_results.sqlite"
|
||||
worker_prefetch_multiplier = 1
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
# pylint: disable=too-many-lines
|
||||
import functools
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from typing import Any, Callable, cast
|
||||
@@ -82,6 +83,8 @@ from superset.commands.dashboard.update import (
|
||||
UpdateDashboardNativeFiltersCommand,
|
||||
)
|
||||
from superset.commands.database.exceptions import DatasetValidationError
|
||||
from superset.commands.distributed_lock.acquire import AcquireDistributedLock
|
||||
from superset.commands.distributed_lock.release import ReleaseDistributedLock
|
||||
from superset.commands.exceptions import TagForbiddenError
|
||||
from superset.commands.importers.exceptions import NoValidFilesFoundError
|
||||
from superset.commands.importers.v1.utils import get_contents_from_bundle
|
||||
@@ -107,6 +110,8 @@ from superset.dashboards.schemas import (
|
||||
DashboardColorsConfigUpdateSchema,
|
||||
DashboardCopySchema,
|
||||
DashboardDatasetSchema,
|
||||
DashboardExportXlsxPostSchema,
|
||||
DashboardExportXlsxResponseSchema,
|
||||
DashboardGetResponseSchema,
|
||||
DashboardNativeFiltersConfigUpdateSchema,
|
||||
DashboardPostSchema,
|
||||
@@ -124,7 +129,9 @@ from superset.dashboards.schemas import (
|
||||
thumbnail_query_schema,
|
||||
)
|
||||
from superset.exceptions import (
|
||||
LockAlreadyHeldException,
|
||||
ScreenshotImageNotAvailableException,
|
||||
SupersetSecurityException,
|
||||
)
|
||||
from superset.extensions import event_logger, security_manager
|
||||
from superset.models.dashboard import Dashboard
|
||||
@@ -135,6 +142,12 @@ from superset.subjects.filters import (
|
||||
FilterRelatedSubjects,
|
||||
subject_type_filter,
|
||||
)
|
||||
from superset.tasks.export_dashboard_excel import (
|
||||
export_dashboard_excel,
|
||||
EXPORT_LOCK_NAMESPACE,
|
||||
export_lock_params,
|
||||
EXPORT_LOCK_TTL_SECONDS,
|
||||
)
|
||||
from superset.tasks.thumbnails import (
|
||||
cache_dashboard_screenshot,
|
||||
cache_dashboard_thumbnail,
|
||||
@@ -274,6 +287,7 @@ class DashboardRestApi(
|
||||
"put_chart_customizations",
|
||||
"put_colors",
|
||||
"export_as_example",
|
||||
"export_xlsx",
|
||||
"list_versions",
|
||||
"get_version",
|
||||
}
|
||||
@@ -291,6 +305,10 @@ class DashboardRestApi(
|
||||
method_permission_name = {
|
||||
**MODEL_API_RW_METHOD_PERMISSION_MAP,
|
||||
"restore": "write",
|
||||
# Reuse the dashboard ``can_export`` permission (the frontend gates the
|
||||
# menu item on it) instead of the ``can_export_xlsx`` FAB would otherwise
|
||||
# derive from the method name.
|
||||
"export_xlsx": "export",
|
||||
}
|
||||
|
||||
# Default list_columns (used if config not set)
|
||||
@@ -496,6 +514,8 @@ class DashboardRestApi(
|
||||
DashboardCopySchema,
|
||||
DashboardGetResponseSchema,
|
||||
DashboardDatasetSchema,
|
||||
DashboardExportXlsxPostSchema,
|
||||
DashboardExportXlsxResponseSchema,
|
||||
TabsPayloadSchema,
|
||||
GetFavStarIdsSchema,
|
||||
EmbeddedDashboardResponseSchema,
|
||||
@@ -1574,6 +1594,132 @@ class DashboardRestApi(
|
||||
response.set_cookie(token, "done", max_age=600)
|
||||
return response
|
||||
|
||||
@expose("/<pk>/export_xlsx/", methods=("POST",))
|
||||
@protect()
|
||||
@safe
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.export_xlsx",
|
||||
log_to_statsd=False,
|
||||
)
|
||||
def export_xlsx(self, pk: int) -> WerkzeugResponse:
|
||||
"""Export all of a dashboard's chart data to an Excel workbook (async).
|
||||
---
|
||||
post:
|
||||
summary: Export dashboard chart data to Excel
|
||||
description: >-
|
||||
Enqueues an async task that writes each chart's data to its own
|
||||
worksheet, uploads the .xlsx to S3, and emails the requesting user a
|
||||
pre-signed download link. Returns immediately with a job id.
|
||||
parameters:
|
||||
- in: path
|
||||
schema:
|
||||
type: integer
|
||||
name: pk
|
||||
description: The dashboard id
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/DashboardExportXlsxPostSchema'
|
||||
responses:
|
||||
202:
|
||||
description: Export task accepted
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/DashboardExportXlsxResponseSchema'
|
||||
400:
|
||||
$ref: '#/components/responses/400'
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
403:
|
||||
$ref: '#/components/responses/403'
|
||||
404:
|
||||
$ref: '#/components/responses/404'
|
||||
500:
|
||||
$ref: '#/components/responses/500'
|
||||
501:
|
||||
description: Excel export is not configured on this server
|
||||
"""
|
||||
if not current_app.config["EXCEL_EXPORT_S3_BUCKET"]:
|
||||
return self.response(
|
||||
501, message="Excel export is not configured on this server."
|
||||
)
|
||||
try:
|
||||
# Tolerate an empty/non-JSON body (e.g. a POST with no Content-Type);
|
||||
# request.json would otherwise raise 415.
|
||||
payload = DashboardExportXlsxPostSchema().load(
|
||||
request.get_json(silent=True) or {}
|
||||
)
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
|
||||
# Image export drives the headless webdriver, so it is only available
|
||||
# when the same screenshot flags the UI checks are enabled. The decorator
|
||||
# form (``@validate_feature_flags``) can't be used here because it would
|
||||
# also block ``mode="data"``; mirror its 404 behavior inline instead.
|
||||
if payload.get("mode") == "images" and not (
|
||||
is_feature_enabled("ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS")
|
||||
and is_feature_enabled("ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT")
|
||||
):
|
||||
return self.response_404()
|
||||
|
||||
dashboard = cast(Dashboard, self.datamodel.get(pk, self._base_filters))
|
||||
if not dashboard:
|
||||
return self.response_404()
|
||||
try:
|
||||
security_manager.raise_for_access(dashboard=dashboard)
|
||||
except SupersetSecurityException:
|
||||
return self.response_403()
|
||||
|
||||
# Email delivery is the only result channel, so an account with an email
|
||||
# address is required; embedded guest users are excluded in this version.
|
||||
if isinstance(g.user, GuestUser) or not getattr(g.user, "email", None):
|
||||
return self.response_400(
|
||||
message="Excel export requires an account with an email address."
|
||||
)
|
||||
if not dashboard.slices:
|
||||
return self.response_400(message="Dashboard has no charts to export.")
|
||||
|
||||
# Throttle: one concurrent export per user+dashboard. Acquire a shared,
|
||||
# atomic distributed lock (Redis when configured, the metadata DB
|
||||
# otherwise) so the guard works across the web server and workers and is
|
||||
# not a no-op under the default cache. The task releases it when it
|
||||
# settles; the TTL is the backstop if that release is ever lost.
|
||||
lock_params = export_lock_params(g.user.id, dashboard.id)
|
||||
try:
|
||||
AcquireDistributedLock(
|
||||
EXPORT_LOCK_NAMESPACE,
|
||||
lock_params,
|
||||
ttl_seconds=EXPORT_LOCK_TTL_SECONDS,
|
||||
).run()
|
||||
except LockAlreadyHeldException:
|
||||
return self.response(
|
||||
202,
|
||||
message="An Excel export for this dashboard is already in progress.",
|
||||
)
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
try:
|
||||
export_dashboard_excel.apply_async(
|
||||
kwargs={
|
||||
"dashboard_id": dashboard.id,
|
||||
"user_id": g.user.id,
|
||||
"active_data_mask": payload.get("active_data_mask", {}),
|
||||
"job_id": job_id,
|
||||
"mode": payload.get("mode", "data"),
|
||||
},
|
||||
task_id=job_id,
|
||||
)
|
||||
except Exception:
|
||||
# If enqueuing fails (e.g. broker down) the task will never run to
|
||||
# release the lock, so free it now rather than block exports until
|
||||
# the TTL expires.
|
||||
ReleaseDistributedLock(EXPORT_LOCK_NAMESPACE, lock_params).run()
|
||||
raise
|
||||
return self.response(202, job_id=job_id)
|
||||
|
||||
@expose("/<pk>/cache_dashboard_screenshot/", methods=("POST",))
|
||||
@validate_feature_flags(["THUMBNAILS", "ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS"])
|
||||
@protect()
|
||||
|
||||
16
superset/dashboards/excel_export/__init__.py
Normal file
16
superset/dashboards/excel_export/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
180
superset/dashboards/excel_export/email.py
Normal file
180
superset/dashboards/excel_export/email.py
Normal file
@@ -0,0 +1,180 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Email rendering and delivery for dashboard Excel exports.
|
||||
|
||||
Bodies use inline styles only (no external CSS, no logo) to match Superset's
|
||||
existing report notification emails, and all user-controlled values (dashboard
|
||||
title, chart names) are HTML-escaped to avoid injection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from flask import current_app
|
||||
from flask_babel import gettext as __, ngettext
|
||||
from markupsafe import escape
|
||||
|
||||
from superset.utils.core import send_email_smtp
|
||||
|
||||
_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
_FOOTER_STYLE = "color:#888;font-size:12px;"
|
||||
_BUTTON_STYLE = (
|
||||
"display:inline-block;padding:10px 16px;background:#20a7c9;color:#ffffff;"
|
||||
"text-decoration:none;border-radius:4px;"
|
||||
)
|
||||
|
||||
# Reason keys under which the export task groups charts it could not export.
|
||||
# The task classifies each omitted chart under one of these; the email renders a
|
||||
# separate, labelled section per non-empty group with its own remediation text.
|
||||
ERROR_NO_QUERY_CONTEXT = "no-query-context"
|
||||
ERROR_GENERAL = "general-exception"
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
return dt.strftime(_DATETIME_FORMAT)
|
||||
|
||||
|
||||
def _humanize_ttl(seconds: int) -> str:
|
||||
"""Render a TTL as a human-readable, pluralized, translatable duration.
|
||||
|
||||
Whole hours read as "24 hours"; sub-hour and non-hour values keep their
|
||||
minutes (e.g. "1 hour 30 minutes", "15 minutes") so the stated lifetime
|
||||
always matches the real pre-signed URL expiration.
|
||||
"""
|
||||
hours, remainder = divmod(seconds, 3600)
|
||||
parts: list[str] = []
|
||||
if hours:
|
||||
parts.append(ngettext("%(num)d hour", "%(num)d hours", hours))
|
||||
if minutes := remainder // 60:
|
||||
parts.append(ngettext("%(num)d minute", "%(num)d minutes", minutes))
|
||||
if not parts:
|
||||
parts.append(ngettext("%(num)d second", "%(num)d seconds", seconds))
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def build_subject(dashboard_title: str, *, success: bool) -> str:
|
||||
"""Build the email subject, prefixed with EMAIL_REPORTS_SUBJECT_PREFIX."""
|
||||
prefix = current_app.config["EMAIL_REPORTS_SUBJECT_PREFIX"]
|
||||
if success:
|
||||
return prefix + __(
|
||||
"Your dashboard export is ready: %(title)s", title=dashboard_title
|
||||
)
|
||||
return prefix + __(
|
||||
"Your dashboard export could not be completed: %(title)s",
|
||||
title=dashboard_title,
|
||||
)
|
||||
|
||||
|
||||
def _errored_section(errored: dict[str, list[str]]) -> str:
|
||||
"""Render one labelled, translated sub-list per non-empty error group.
|
||||
|
||||
``errored`` maps a reason key (see the ``ERROR_*`` constants) to the labels
|
||||
of the charts that were omitted for that reason. Known reasons are rendered
|
||||
first, in a stable order, each with its own remediation text; any unknown
|
||||
reason key falls back to a generic message so nothing is silently dropped.
|
||||
"""
|
||||
if not errored:
|
||||
return ""
|
||||
notes = {
|
||||
ERROR_NO_QUERY_CONTEXT: __(
|
||||
"The following charts were omitted because they have no saved query "
|
||||
"context. To include them, open each chart in Explore and re-save."
|
||||
),
|
||||
ERROR_GENERAL: __(
|
||||
"The following charts were omitted because an error occurred while "
|
||||
"exporting them:"
|
||||
),
|
||||
}
|
||||
fallback = __("The following charts could not be exported:")
|
||||
ordered = [ERROR_NO_QUERY_CONTEXT, ERROR_GENERAL]
|
||||
reasons = ordered + [reason for reason in errored if reason not in ordered]
|
||||
sections = []
|
||||
for reason in reasons:
|
||||
labels = errored.get(reason)
|
||||
if not labels:
|
||||
continue
|
||||
note = notes.get(reason, fallback)
|
||||
items = "".join(f"<li>{escape(label)}</li>" for label in labels)
|
||||
sections.append(f"<p>{note}</p><ul>{items}</ul>")
|
||||
return "".join(sections)
|
||||
|
||||
|
||||
def build_success_email(
|
||||
dashboard_title: str,
|
||||
download_url: str,
|
||||
requested_at: datetime,
|
||||
expires_at: datetime,
|
||||
ttl_seconds: int,
|
||||
errored: dict[str, list[str]],
|
||||
) -> str:
|
||||
"""Render the success email body (HTML)."""
|
||||
title = escape(dashboard_title)
|
||||
url = escape(download_url)
|
||||
ready = __('Your export of "%(title)s" is ready.', title=title)
|
||||
button = __("Download Excel file")
|
||||
expiry = __(
|
||||
"This link expires in %(duration)s (%(when)s UTC).",
|
||||
duration=_humanize_ttl(ttl_seconds),
|
||||
when=_fmt(expires_at),
|
||||
)
|
||||
requested = __(
|
||||
"This export was requested on %(when)s UTC.", when=_fmt(requested_at)
|
||||
)
|
||||
disclaimer = __("If you did not request this, you can ignore this email.")
|
||||
return (
|
||||
'<html><body style="font-family:Arial,sans-serif;color:#333;">'
|
||||
f"<p>{ready}</p>"
|
||||
f'<p><a href="{url}" style="{_BUTTON_STYLE}">{button}</a></p>'
|
||||
f"<p>{expiry}</p>"
|
||||
f"{_errored_section(errored)}"
|
||||
"<hr/>"
|
||||
f'<p style="{_FOOTER_STYLE}">{requested}<br/>{disclaimer}</p>'
|
||||
"</body></html>"
|
||||
)
|
||||
|
||||
|
||||
def build_failure_email(dashboard_title: str, requested_at: datetime) -> str:
|
||||
"""Render the failure email body (HTML)."""
|
||||
title = escape(dashboard_title)
|
||||
failed = __('Your export of "%(title)s" could not be completed.', title=title)
|
||||
advice = __(
|
||||
"An error occurred while generating the file. Please try again, or "
|
||||
"contact your administrator if the problem persists."
|
||||
)
|
||||
requested = __(
|
||||
"This export was requested on %(when)s UTC.", when=_fmt(requested_at)
|
||||
)
|
||||
return (
|
||||
'<html><body style="font-family:Arial,sans-serif;color:#333;">'
|
||||
f"<p>{failed}</p>"
|
||||
f"<p>{advice}</p>"
|
||||
"<hr/>"
|
||||
f'<p style="{_FOOTER_STYLE}">{requested}</p>'
|
||||
"</body></html>"
|
||||
)
|
||||
|
||||
|
||||
def send_export_email(to: str, subject: str, html_content: str) -> None:
|
||||
"""Send an export email via the configured SMTP transport."""
|
||||
send_email_smtp(
|
||||
to=to,
|
||||
subject=subject,
|
||||
html_content=html_content,
|
||||
config=current_app.config,
|
||||
)
|
||||
97
superset/dashboards/excel_export/layout.py
Normal file
97
superset/dashboards/excel_export/layout.py
Normal file
@@ -0,0 +1,97 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Determine the order in which a dashboard's charts appear in its layout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.models.dashboard import Dashboard
|
||||
from superset.models.slice import Slice
|
||||
|
||||
CHART_TYPE = "CHART"
|
||||
ROOT_ID = "ROOT_ID"
|
||||
|
||||
|
||||
def _walk_chart_ids(position: dict[str, Any]) -> list[int]:
|
||||
"""
|
||||
Depth-first walk of a dashboard ``position_json`` returning chart ids in
|
||||
visual (layout) order, including tab-nested charts. Each chart id appears
|
||||
once (first occurrence wins); cycles are guarded against.
|
||||
"""
|
||||
if ROOT_ID not in position:
|
||||
return []
|
||||
|
||||
ordered: list[int] = []
|
||||
seen_charts: set[int] = set()
|
||||
visited_nodes: set[str] = set()
|
||||
stack: list[str] = [ROOT_ID]
|
||||
|
||||
while stack:
|
||||
node_id = stack.pop()
|
||||
if node_id in visited_nodes:
|
||||
continue
|
||||
visited_nodes.add(node_id)
|
||||
|
||||
node = position.get(node_id)
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
if node.get("type") == CHART_TYPE:
|
||||
chart_id = node.get("meta", {}).get("chartId")
|
||||
if isinstance(chart_id, int) and chart_id not in seen_charts:
|
||||
seen_charts.add(chart_id)
|
||||
ordered.append(chart_id)
|
||||
|
||||
# Push children in reverse so they are popped in their declared order.
|
||||
children = node.get("children", [])
|
||||
for child_id in reversed(children):
|
||||
stack.append(child_id)
|
||||
|
||||
return ordered
|
||||
|
||||
|
||||
def get_charts_in_layout_order(dashboard: Dashboard) -> list[Slice]:
|
||||
"""
|
||||
Return the dashboard's charts ordered by their position in the layout.
|
||||
|
||||
Charts are visited depth-first over ``position_json`` (so tab-nested charts
|
||||
are included in tab order), de-duplicated when the same chart is placed more
|
||||
than once, and any chart that belongs to the dashboard but is absent from
|
||||
the layout is appended at the end ordered by id. Layout entries that no
|
||||
longer correspond to a dashboard chart are skipped.
|
||||
|
||||
:param dashboard: The dashboard whose charts to order
|
||||
:returns: The dashboard's :class:`Slice` objects in layout order
|
||||
"""
|
||||
slices_by_id: dict[int, Slice] = {slc.id: slc for slc in dashboard.slices}
|
||||
|
||||
result: list[Slice] = []
|
||||
used: set[int] = set()
|
||||
for chart_id in _walk_chart_ids(dashboard.position):
|
||||
slc = slices_by_id.get(chart_id)
|
||||
if slc is not None and chart_id not in used:
|
||||
used.add(chart_id)
|
||||
result.append(slc)
|
||||
|
||||
orphans = sorted(
|
||||
(slc for chart_id, slc in slices_by_id.items() if chart_id not in used),
|
||||
key=lambda slc: slc.id,
|
||||
)
|
||||
result.extend(orphans)
|
||||
return result
|
||||
103
superset/dashboards/excel_export/screenshot.py
Normal file
103
superset/dashboards/excel_export/screenshot.py
Normal file
@@ -0,0 +1,103 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Render a single dashboard chart to a PNG for the image-mode Excel export.
|
||||
|
||||
This reuses the same headless render path scheduled reports use
|
||||
(:class:`~superset.utils.screenshots.ChartScreenshot`), but points it at an
|
||||
Explore URL whose ``form_data`` carries the live dashboard filter state — so an
|
||||
embedded image reflects the same filters the data path applies, rather than the
|
||||
chart's default saved state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from flask import current_app
|
||||
|
||||
from superset.charts.data.dashboard_filter_context import (
|
||||
get_dashboard_filter_context,
|
||||
)
|
||||
from superset.utils import json
|
||||
from superset.utils.screenshots import ChartScreenshot
|
||||
from superset.utils.urls import get_url_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def render_chart_image(
|
||||
chart: Any,
|
||||
dashboard_id: int,
|
||||
active_data_mask: dict[str, Any],
|
||||
user: Any,
|
||||
) -> bytes | None:
|
||||
"""
|
||||
Render ``chart`` (as seen on ``dashboard_id``) to PNG bytes.
|
||||
|
||||
The chart is rendered through Explore in standalone mode with the live
|
||||
dashboard filter state injected as ``extra_form_data`` — the same object the
|
||||
data path merges into the query context — so the image and the data stay
|
||||
consistent.
|
||||
|
||||
:param chart: The ``Slice`` to render
|
||||
:param dashboard_id: The dashboard the chart is displayed on (for filter scope)
|
||||
:param active_data_mask: Live dashboard filter state keyed by native filter id
|
||||
:param user: The requesting user; the render runs with their permissions
|
||||
:returns: PNG bytes, or ``None`` if the render failed (the caller skips and
|
||||
notes the chart)
|
||||
"""
|
||||
try:
|
||||
filter_context = get_dashboard_filter_context(
|
||||
dashboard_id=dashboard_id,
|
||||
chart_id=chart.id,
|
||||
active_data_mask=active_data_mask,
|
||||
)
|
||||
|
||||
# Start from the chart's saved form data and force the slice id, then
|
||||
# layer the live filters on top so the render matches the data path.
|
||||
form_data: dict[str, Any] = json.loads(chart.params or "{}")
|
||||
form_data["slice_id"] = chart.id
|
||||
if filter_context.extra_form_data:
|
||||
form_data["extra_form_data"] = filter_context.extra_form_data
|
||||
|
||||
url = get_url_path(
|
||||
"ExploreView.root",
|
||||
form_data=json.dumps(form_data),
|
||||
)
|
||||
|
||||
window_size = current_app.config["WEBDRIVER_WINDOW"]["slice"]
|
||||
screenshot = ChartScreenshot(
|
||||
url,
|
||||
chart.digest,
|
||||
window_size=window_size,
|
||||
thumb_size=window_size,
|
||||
)
|
||||
return screenshot.get_screenshot(user=user)
|
||||
except SoftTimeLimitExceeded:
|
||||
# A soft timeout aborts the whole export; don't let the broad handler
|
||||
# below turn it into a ``None`` (a per-chart "could not render") result.
|
||||
raise
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception(
|
||||
"Failed to render image for chart %s in dashboard %s",
|
||||
getattr(chart, "id", "?"),
|
||||
dashboard_id,
|
||||
)
|
||||
return None
|
||||
@@ -18,7 +18,7 @@ import re
|
||||
from typing import Any, Mapping, Union
|
||||
|
||||
from marshmallow import fields, post_dump, post_load, pre_load, Schema
|
||||
from marshmallow.validate import Length, ValidationError
|
||||
from marshmallow.validate import Length, OneOf, ValidationError
|
||||
|
||||
from superset import security_manager
|
||||
from superset.subjects.schemas import SubjectResponseSchema
|
||||
@@ -628,3 +628,30 @@ class CacheScreenshotSchema(Schema):
|
||||
fields.List(fields.Str(), validate=lambda x: len(x) == 2), required=False
|
||||
)
|
||||
permalinkKey = fields.Str(required=False) # noqa: N815
|
||||
|
||||
|
||||
class DashboardExportXlsxPostSchema(Schema):
|
||||
active_data_mask = fields.Dict(
|
||||
keys=fields.Str(),
|
||||
values=fields.Dict(),
|
||||
load_default=dict,
|
||||
metadata={
|
||||
"description": "Live dashboard filter state keyed by native filter id, "
|
||||
"each carrying an extraFormData object."
|
||||
},
|
||||
)
|
||||
mode = fields.String(
|
||||
load_default="data",
|
||||
validate=OneOf(["data", "images"]),
|
||||
metadata={
|
||||
"description": "Export mode: 'data' streams each chart's tabular result "
|
||||
"(default); 'images' embeds non-table charts as rendered images and "
|
||||
"keeps table charts tabular."
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class DashboardExportXlsxResponseSchema(Schema):
|
||||
job_id = fields.String(
|
||||
metadata={"description": "Correlation id for the async export task"}
|
||||
)
|
||||
|
||||
361
superset/tasks/export_dashboard_excel.py
Normal file
361
superset/tasks/export_dashboard_excel.py
Normal file
@@ -0,0 +1,361 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Celery task that exports every chart on a dashboard to a single multi-sheet
|
||||
``.xlsx`` file, uploads it to S3, and emails the requesting user a pre-signed
|
||||
download link.
|
||||
|
||||
In ``"data"`` mode the task re-runs each chart's saved query context under the
|
||||
requesting user, applies the live dashboard filter state, and streams the results
|
||||
row-by-row into a constant-memory workbook so large dashboards never load all
|
||||
data at once. In ``"images"`` mode non-table charts are instead rendered to
|
||||
images (through the same headless path as scheduled reports, reflecting the live
|
||||
filters) and embedded, while table-like charts stay tabular.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from flask import current_app, g
|
||||
|
||||
from superset import db, security_manager
|
||||
from superset.charts.data.dashboard_filter_context import (
|
||||
apply_dashboard_filter_context,
|
||||
get_dashboard_filter_context,
|
||||
)
|
||||
from superset.charts.schemas import ChartDataQueryContextSchema
|
||||
from superset.commands.chart.data.get_data_command import ChartDataCommand
|
||||
from superset.commands.distributed_lock.release import ReleaseDistributedLock
|
||||
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
|
||||
from superset.dashboards.excel_export import email
|
||||
from superset.dashboards.excel_export.layout import get_charts_in_layout_order
|
||||
from superset.dashboards.excel_export.screenshot import render_chart_image
|
||||
from superset.extensions import celery_app
|
||||
from superset.utils import json, s3
|
||||
from superset.utils.core import override_user
|
||||
from superset.utils.excel_streaming import StreamingXlsxWriter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Export modes: "data" streams every chart's tabular result (the default,
|
||||
# unchanged behavior); "images" embeds non-table charts as rendered images and
|
||||
# keeps only table-like charts tabular.
|
||||
EXPORT_MODE_DATA = "data"
|
||||
EXPORT_MODE_IMAGES = "images"
|
||||
|
||||
# Viz types kept as tabular data in image mode; everything else is rendered as an
|
||||
# image. Operators can override the set via ``EXCEL_EXPORT_TABLE_VIZ_TYPES``.
|
||||
TABLE_VIZ_TYPES = {"table", "pivot_table_v2", "pivot_table"}
|
||||
|
||||
EXPORT_SOFT_TIME_LIMIT = 600
|
||||
EXPORT_HARD_TIME_LIMIT = 660
|
||||
|
||||
# Namespace + TTL for the per-user+dashboard in-flight lock the API acquires
|
||||
# before enqueue and this task releases when it settles. The lock uses the
|
||||
# shared, atomic DistributedLock backend (Redis when configured, the metadata
|
||||
# DB otherwise) so it actually synchronizes across the web server and workers —
|
||||
# unlike a plain cache, which is a no-op under the default ``NullCache``.
|
||||
# The TTL outlives the hard time limit so a worker killed at that limit (which
|
||||
# skips the ``finally`` release) cannot hold the lock forever; the release in
|
||||
# ``finally`` is the fast path that frees it as soon as the task settles.
|
||||
EXPORT_LOCK_NAMESPACE = "excel_export"
|
||||
EXPORT_LOCK_TTL_SECONDS = EXPORT_HARD_TIME_LIMIT + 60
|
||||
|
||||
|
||||
def export_lock_params(user_id: int, dashboard_id: int) -> dict[str, int]:
|
||||
"""Key parameters identifying the per-user+dashboard in-flight lock."""
|
||||
return {"user_id": user_id, "dashboard_id": dashboard_id}
|
||||
|
||||
|
||||
class _ChartSkippedError(Exception):
|
||||
"""Signals a chart that could not be exported and should be listed as skipped."""
|
||||
|
||||
|
||||
def _chart_label(chart: Any) -> str:
|
||||
"""Human-readable label for a chart in the skipped-charts list."""
|
||||
return f"{chart.id} - {chart.slice_name or ''}".strip()
|
||||
|
||||
|
||||
def _record_to_row(record: dict[str, Any], colnames: list[str]) -> list[Any]:
|
||||
return [record.get(col) for col in colnames]
|
||||
|
||||
|
||||
def _table_viz_types() -> set[str]:
|
||||
"""Viz types kept tabular in image mode (config override or built-in default)."""
|
||||
return current_app.config.get("EXCEL_EXPORT_TABLE_VIZ_TYPES") or TABLE_VIZ_TYPES
|
||||
|
||||
|
||||
def _renders_as_image(chart: Any, mode: str) -> bool:
|
||||
"""Whether this chart is embedded as an image rather than streamed as data."""
|
||||
return mode == EXPORT_MODE_IMAGES and chart.viz_type not in _table_viz_types()
|
||||
|
||||
|
||||
def _write_chart_image_sheet(
|
||||
writer: StreamingXlsxWriter,
|
||||
chart: Any,
|
||||
dashboard_id: int,
|
||||
active_data_mask: dict[str, Any],
|
||||
user: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Render a single chart to an image and embed it as its own sheet.
|
||||
|
||||
:raises _ChartSkippedError: if the chart could not be rendered
|
||||
"""
|
||||
image = render_chart_image(chart, dashboard_id, active_data_mask, user)
|
||||
if image is None:
|
||||
raise _ChartSkippedError
|
||||
writer.add_image_sheet(_chart_label(chart), image)
|
||||
|
||||
|
||||
def _write_chart_sheets(
|
||||
writer: StreamingXlsxWriter,
|
||||
chart: Any,
|
||||
dashboard_id: int,
|
||||
active_data_mask: dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Run a single chart's query and stream its result(s) into the workbook.
|
||||
|
||||
Charts may yield more than one query (e.g. mixed-series charts); each becomes
|
||||
its own sheet. Raises if the chart cannot be exported, so the caller can skip
|
||||
it and note it in the email.
|
||||
"""
|
||||
json_body = json.loads(chart.query_context)
|
||||
# Override any stale saved values: we always want full JSON results.
|
||||
json_body["result_format"] = ChartDataResultFormat.JSON
|
||||
json_body["result_type"] = ChartDataResultType.FULL
|
||||
json_body.pop("force", None)
|
||||
|
||||
filter_context = get_dashboard_filter_context(
|
||||
dashboard_id=dashboard_id,
|
||||
chart_id=chart.id,
|
||||
active_data_mask=active_data_mask,
|
||||
)
|
||||
if filter_context.extra_form_data:
|
||||
apply_dashboard_filter_context(json_body, filter_context.extra_form_data)
|
||||
|
||||
# Jinja macros resolve form data from g.form_data; expose the saved context.
|
||||
g.form_data = json_body
|
||||
|
||||
query_context = ChartDataQueryContextSchema().load(json_body)
|
||||
command = ChartDataCommand(query_context)
|
||||
command.validate()
|
||||
result = command.run()
|
||||
|
||||
for index, query in enumerate(result["queries"]):
|
||||
colnames = query.get("colnames") or []
|
||||
data = query.get("data") or []
|
||||
if index == 0:
|
||||
name = f"{chart.id} - {chart.slice_name or ''}"
|
||||
else:
|
||||
name = f"{chart.id}.{index} - {chart.slice_name or ''}"
|
||||
writer.add_sheet(
|
||||
name,
|
||||
colnames,
|
||||
(_record_to_row(record, colnames) for record in data),
|
||||
)
|
||||
|
||||
|
||||
def _build_workbook(
|
||||
path: str,
|
||||
dashboard: Any,
|
||||
active_data_mask: dict[str, Any],
|
||||
job_id: str,
|
||||
mode: str,
|
||||
user: Any,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Build the workbook on disk.
|
||||
|
||||
Return the charts that could not be exported, grouped by the reason they
|
||||
were omitted (see the ``email.ERROR_*`` reason keys), so the notification
|
||||
can explain each group separately.
|
||||
"""
|
||||
errored: dict[str, list[str]] = {}
|
||||
writer = StreamingXlsxWriter(path)
|
||||
try:
|
||||
for chart in get_charts_in_layout_order(dashboard):
|
||||
label = _chart_label(chart)
|
||||
as_image = _renders_as_image(chart, mode)
|
||||
# Image charts render from their saved params and don't need a query
|
||||
# context; data (and table) charts still do.
|
||||
if not as_image and not chart.query_context:
|
||||
errored.setdefault(email.ERROR_NO_QUERY_CONTEXT, []).append(label)
|
||||
continue
|
||||
try:
|
||||
if as_image:
|
||||
_write_chart_image_sheet(
|
||||
writer, chart, dashboard.id, active_data_mask, user
|
||||
)
|
||||
else:
|
||||
_write_chart_sheets(writer, chart, dashboard.id, active_data_mask)
|
||||
except SoftTimeLimitExceeded:
|
||||
# A soft timeout is a task-level signal, not a per-chart failure:
|
||||
# let it propagate so the outer handler emails a failure and runs
|
||||
# cleanup, rather than continuing until the hard limit kills the
|
||||
# worker (which would skip cleanup, leak temp files, and hold the
|
||||
# in-flight lock until its TTL). ``except Exception`` below would
|
||||
# otherwise swallow it, since it subclasses ``Exception``.
|
||||
raise
|
||||
except _ChartSkippedError:
|
||||
logger.warning(
|
||||
"Skipping chart %s in dashboard export %s (could not render)",
|
||||
chart.id,
|
||||
job_id,
|
||||
)
|
||||
errored.setdefault(email.ERROR_GENERAL, []).append(label)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception(
|
||||
"Skipping chart %s in dashboard export %s", chart.id, job_id
|
||||
)
|
||||
errored.setdefault(email.ERROR_GENERAL, []).append(label)
|
||||
|
||||
if writer.sheet_count == 0:
|
||||
flat = [label for labels in errored.values() for label in labels]
|
||||
writer.add_summary_sheet(
|
||||
"Export Summary",
|
||||
["No chart data could be exported.", *flat],
|
||||
)
|
||||
finally:
|
||||
writer.close()
|
||||
return errored
|
||||
|
||||
|
||||
def _send_failure_email(
|
||||
user: Any, dashboard_title: str, requested_at: datetime
|
||||
) -> None:
|
||||
if not (user and getattr(user, "email", None)):
|
||||
return
|
||||
try:
|
||||
email.send_export_email(
|
||||
user.email,
|
||||
email.build_subject(dashboard_title, success=False),
|
||||
email.build_failure_email(dashboard_title, requested_at),
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception("Failed to send export failure email")
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="export_dashboard_excel",
|
||||
bind=True,
|
||||
soft_time_limit=EXPORT_SOFT_TIME_LIMIT,
|
||||
time_limit=EXPORT_HARD_TIME_LIMIT,
|
||||
max_retries=0,
|
||||
)
|
||||
def export_dashboard_excel(
|
||||
self: Any, # pylint: disable=unused-argument
|
||||
dashboard_id: int,
|
||||
user_id: int,
|
||||
active_data_mask: dict[str, Any],
|
||||
job_id: str,
|
||||
mode: str = EXPORT_MODE_DATA,
|
||||
) -> None:
|
||||
"""
|
||||
Export a dashboard's charts to an ``.xlsx`` and email a download link.
|
||||
|
||||
:param dashboard_id: The dashboard to export
|
||||
:param user_id: The requesting user (the task runs with their permissions)
|
||||
:param active_data_mask: Live dashboard filter state keyed by native filter id
|
||||
:param job_id: Correlation id, also the Celery task id and S3 object name
|
||||
:param mode: ``"data"`` streams every chart's tabular result; ``"images"``
|
||||
embeds non-table charts as rendered images and keeps tables tabular
|
||||
"""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
requested_at = datetime.now(tz=timezone.utc)
|
||||
user = security_manager.get_user_by_id(user_id)
|
||||
dashboard_title = ""
|
||||
tmp_path: str | None = None
|
||||
|
||||
try:
|
||||
with override_user(user, force=False):
|
||||
dashboard = (
|
||||
db.session.query(Dashboard).filter_by(id=dashboard_id).one_or_none()
|
||||
)
|
||||
if dashboard is None:
|
||||
raise ValueError(f"Dashboard {dashboard_id} not found")
|
||||
dashboard_title = dashboard.dashboard_title or f"Dashboard {dashboard_id}"
|
||||
|
||||
file_descriptor, tmp_path = tempfile.mkstemp(
|
||||
suffix=".xlsx", prefix=f"dash-export-{job_id}-"
|
||||
)
|
||||
os.close(file_descriptor)
|
||||
|
||||
errored = _build_workbook(
|
||||
tmp_path, dashboard, active_data_mask, job_id, mode, user
|
||||
)
|
||||
|
||||
bucket = current_app.config["EXCEL_EXPORT_S3_BUCKET"]
|
||||
key = (
|
||||
f"{current_app.config['EXCEL_EXPORT_S3_KEY_PREFIX']}"
|
||||
f"{dashboard_id}/{job_id}.xlsx"
|
||||
)
|
||||
ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"]
|
||||
|
||||
s3.upload_file_to_s3(tmp_path, bucket, key)
|
||||
download_url = s3.generate_presigned_url(bucket, key, ttl)
|
||||
expires_at = datetime.now(tz=timezone.utc) + timedelta(seconds=ttl)
|
||||
|
||||
if user and getattr(user, "email", None):
|
||||
try:
|
||||
email.send_export_email(
|
||||
user.email,
|
||||
email.build_subject(dashboard_title, success=True),
|
||||
email.build_success_email(
|
||||
dashboard_title=dashboard_title,
|
||||
download_url=download_url,
|
||||
requested_at=requested_at,
|
||||
expires_at=expires_at,
|
||||
ttl_seconds=ttl,
|
||||
errored=errored,
|
||||
),
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# The file is already in S3; a send failure should not trigger
|
||||
# a misleading failure email.
|
||||
logger.exception("Failed to send export success email")
|
||||
except SoftTimeLimitExceeded:
|
||||
logger.warning("Dashboard excel export %s timed out", job_id)
|
||||
_send_failure_email(user, dashboard_title, requested_at)
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Dashboard excel export %s failed", job_id)
|
||||
_send_failure_email(user, dashboard_title, requested_at)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
ReleaseDistributedLock(
|
||||
EXPORT_LOCK_NAMESPACE,
|
||||
export_lock_params(user_id, dashboard_id),
|
||||
).run()
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# Best-effort: the lock's TTL is the backstop if this fails.
|
||||
logger.exception(
|
||||
"Failed to release in-flight export lock for user %s dashboard %s",
|
||||
user_id,
|
||||
dashboard_id,
|
||||
)
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
@@ -41,18 +41,21 @@ NEUTRAL_DOCUMENT_PROPERTIES: dict[str, Any] = {
|
||||
"created": NEUTRAL_TIMESTAMP,
|
||||
}
|
||||
|
||||
# Leading characters that turn a cell into a formula in spreadsheet apps. Shared
|
||||
# with the streaming writer (superset.utils.excel_streaming) so both export paths
|
||||
# guard against the same formula-injection vectors.
|
||||
FORMULA_PREFIXES = {"=", "+", "-", "@"}
|
||||
|
||||
|
||||
def quote_formulas(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Make sure to quote any formulas for security reasons.
|
||||
"""
|
||||
formula_prefixes = {"=", "+", "-", "@"}
|
||||
|
||||
for col in df.select_dtypes(include="object").columns:
|
||||
df[col] = df[col].apply(
|
||||
lambda x: (
|
||||
f"'{x}"
|
||||
if isinstance(x, str) and len(x) and x[0] in formula_prefixes
|
||||
if isinstance(x, str) and len(x) and x[0] in FORMULA_PREFIXES
|
||||
else x
|
||||
)
|
||||
)
|
||||
|
||||
250
superset/utils/excel_streaming.py
Normal file
250
superset/utils/excel_streaming.py
Normal file
@@ -0,0 +1,250 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Streaming XLSX writer for multi-sheet dashboard exports.
|
||||
|
||||
Unlike :mod:`superset.utils.excel`, which builds an in-memory DataFrame per
|
||||
sheet and hands the whole thing to ``xlsxwriter`` at once, this writer opens the
|
||||
workbook in ``constant_memory`` mode and writes rows one at a time, so
|
||||
``xlsxwriter`` keeps at most one row per sheet buffered on the writer side. The
|
||||
source records may still be materialized upstream (e.g. by the chart query
|
||||
response); this bounds only the writer's own footprint, not the caller's.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import numbers
|
||||
import re
|
||||
from collections.abc import Iterable, Sequence
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import xlsxwriter
|
||||
|
||||
from superset.utils.excel import FORMULA_PREFIXES, NEUTRAL_DOCUMENT_PROPERTIES
|
||||
|
||||
# Excel limits a sheet name to 31 characters and forbids these characters.
|
||||
MAX_SHEET_NAME_LEN = 31
|
||||
_INVALID_SHEET_CHARS_RE = re.compile(r"[\[\]:*?/\\]")
|
||||
# Excel reserves the sheet name "History" (case-insensitive).
|
||||
_RESERVED_SHEET_NAME = "history"
|
||||
|
||||
# A worksheet holds at most 1,048,576 rows; one is reserved for the header.
|
||||
MAX_DATA_ROWS_PER_SHEET = 1_048_576 - 1
|
||||
|
||||
# Excel cannot represent integers beyond 10**15 without precision loss.
|
||||
_MAX_EXCEL_INT = 10**15
|
||||
|
||||
|
||||
def _quote_if_formula(text: str) -> str:
|
||||
"""
|
||||
Prefix formula-like text with an apostrophe so spreadsheet apps treat it as
|
||||
literal text (defense against formula injection).
|
||||
|
||||
Leading whitespace is ignored when detecting a formula, because spreadsheet
|
||||
apps still evaluate a cell whose formula prefix is preceded by spaces or
|
||||
tabs (e.g. ``" =cmd"`` or ``"\\t=cmd"``).
|
||||
"""
|
||||
stripped = text.lstrip()
|
||||
return f"'{text}" if stripped and stripped[0] in FORMULA_PREFIXES else text
|
||||
|
||||
|
||||
def _coerce_float_cell(value: Any) -> Any:
|
||||
"""
|
||||
Convert a ``Decimal``/real value to something ``xlsxwriter`` accepts.
|
||||
|
||||
``float()`` on a non-finite ``Decimal`` ("NaN"/"Infinity") yields a value
|
||||
xlsxwriter rejects, and an over-large value can raise ``OverflowError``;
|
||||
blank the former and stringify the latter, and stringify magnitudes Excel
|
||||
cannot represent precisely.
|
||||
"""
|
||||
try:
|
||||
number = float(value)
|
||||
except (OverflowError, ValueError):
|
||||
return str(value)
|
||||
if not math.isfinite(number):
|
||||
return ""
|
||||
return str(number) if abs(number) > _MAX_EXCEL_INT else number
|
||||
|
||||
|
||||
def sanitize_sheet_name(raw: str, used: set[str]) -> str:
|
||||
"""
|
||||
Produce a valid, unique Excel sheet name from ``raw``.
|
||||
|
||||
Replaces forbidden characters, strips surrounding apostrophes/whitespace,
|
||||
avoids the reserved name "History", truncates to 31 characters, and
|
||||
disambiguates case-insensitive collisions with ``~2``/``~3`` suffixes.
|
||||
The chosen name (lower-cased) is added to ``used``.
|
||||
|
||||
:param raw: The desired sheet name (e.g. ``"42 - Sales by Region"``)
|
||||
:param used: Lower-cased names already taken; mutated with the result
|
||||
:returns: A sanitized, unique sheet name no longer than 31 characters
|
||||
"""
|
||||
name = _INVALID_SHEET_CHARS_RE.sub("_", raw or "")
|
||||
name = name.strip().strip("'").strip()
|
||||
if not name:
|
||||
name = "Sheet"
|
||||
if name.lower() == _RESERVED_SHEET_NAME:
|
||||
name = f"{name}_"
|
||||
name = name[:MAX_SHEET_NAME_LEN]
|
||||
|
||||
if name.lower() not in used:
|
||||
used.add(name.lower())
|
||||
return name
|
||||
|
||||
suffix = 2
|
||||
while True:
|
||||
marker = f"~{suffix}"
|
||||
candidate = name[: MAX_SHEET_NAME_LEN - len(marker)] + marker
|
||||
if candidate.lower() not in used:
|
||||
used.add(candidate.lower())
|
||||
return candidate
|
||||
suffix += 1
|
||||
|
||||
|
||||
def _sanitize_cell(value: Any) -> Any:
|
||||
"""
|
||||
Coerce a single cell value into something safe for ``xlsxwriter``.
|
||||
|
||||
Quotes formula-like strings (defense against formula injection), stringifies
|
||||
integers/floats Excel cannot represent precisely, renders temporal values as
|
||||
ISO strings (timezones are not natively supported), and blanks out ``None``
|
||||
and non-finite floats.
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
# bool is a subclass of int; preserve it before the numeric branches.
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return _quote_if_formula(value)
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Decimal):
|
||||
return _coerce_float_cell(value)
|
||||
if isinstance(value, numbers.Integral):
|
||||
number = int(value)
|
||||
return str(number) if abs(number) > _MAX_EXCEL_INT else number
|
||||
if isinstance(value, numbers.Real):
|
||||
return _coerce_float_cell(value)
|
||||
# Anything else (lists, dicts, custom objects) is stringified, still guarding
|
||||
# against formula injection on the resulting text.
|
||||
return _quote_if_formula(str(value))
|
||||
|
||||
|
||||
class StreamingXlsxWriter:
|
||||
"""
|
||||
A thin wrapper over ``xlsxwriter`` in constant-memory mode that writes one
|
||||
sheet per chart, row by row.
|
||||
|
||||
Sheet names are sanitized and de-duplicated, cell values are sanitized for
|
||||
safety/compatibility, and per-sheet row counts are capped at Excel's limit.
|
||||
Always call :meth:`close` (e.g. in a ``finally`` block) to finalize the file.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self._workbook = xlsxwriter.Workbook(path, {"constant_memory": True})
|
||||
# Reset document properties so the file carries no identifying details.
|
||||
self._workbook.set_properties(NEUTRAL_DOCUMENT_PROPERTIES)
|
||||
self._used_sheet_names: set[str] = set()
|
||||
self.sheet_count = 0
|
||||
|
||||
def add_sheet(
|
||||
self,
|
||||
name: str,
|
||||
columns: Sequence[Any],
|
||||
rows: Iterable[Sequence[Any]],
|
||||
) -> int:
|
||||
"""
|
||||
Write a header row followed by data rows into a new sheet.
|
||||
|
||||
:param name: Desired sheet name (sanitized/de-duplicated automatically)
|
||||
:param columns: Column headers
|
||||
:param rows: Iterable of row sequences, streamed one at a time
|
||||
:returns: The number of data rows actually written (capped just below
|
||||
Excel's per-sheet limit; when the data is larger a final notice row
|
||||
is appended and the dropped rows are not counted)
|
||||
"""
|
||||
sheet_name = sanitize_sheet_name(name, self._used_sheet_names)
|
||||
worksheet = self._workbook.add_worksheet(sheet_name)
|
||||
worksheet.write_row(0, 0, [_sanitize_cell(col) for col in columns])
|
||||
|
||||
# Reserve the final row for a truncation notice, so when the data
|
||||
# exceeds the sheet's capacity the user can see rows were dropped
|
||||
# instead of silently losing them.
|
||||
row_cap = MAX_DATA_ROWS_PER_SHEET - 1
|
||||
written = 0
|
||||
truncated = False
|
||||
for row in rows:
|
||||
if written >= row_cap:
|
||||
truncated = True
|
||||
break
|
||||
worksheet.write_row(written + 1, 0, [_sanitize_cell(cell) for cell in row])
|
||||
written += 1
|
||||
|
||||
if truncated:
|
||||
worksheet.write_string(
|
||||
written + 1,
|
||||
0,
|
||||
f"[Truncated: only first {written:,} rows exported]",
|
||||
)
|
||||
|
||||
self.sheet_count += 1
|
||||
return written
|
||||
|
||||
def add_image_sheet(self, name: str, image_bytes: bytes) -> None:
|
||||
"""
|
||||
Write a single sheet holding a rendered chart image.
|
||||
|
||||
The image is embedded top-left; ``xlsxwriter`` buffers image data and
|
||||
writes it at :meth:`close`, so this composes with ``constant_memory``
|
||||
mode just like the row-streaming sheets.
|
||||
|
||||
:param name: Desired sheet name (sanitized/de-duplicated automatically)
|
||||
:param image_bytes: PNG bytes to embed
|
||||
"""
|
||||
sheet_name = sanitize_sheet_name(name, self._used_sheet_names)
|
||||
worksheet = self._workbook.add_worksheet(sheet_name)
|
||||
worksheet.insert_image(
|
||||
0,
|
||||
0,
|
||||
f"{sheet_name}.png",
|
||||
{"image_data": BytesIO(image_bytes)},
|
||||
)
|
||||
self.sheet_count += 1
|
||||
|
||||
def add_summary_sheet(self, name: str, lines: Sequence[str]) -> None:
|
||||
"""
|
||||
Write a single-column informational sheet (e.g. a list of skipped charts).
|
||||
|
||||
Lines are written as string cells, so formula-like text is never executed.
|
||||
"""
|
||||
sheet_name = sanitize_sheet_name(name, self._used_sheet_names)
|
||||
worksheet = self._workbook.add_worksheet(sheet_name)
|
||||
for index, line in enumerate(lines):
|
||||
worksheet.write_string(index, 0, str(line))
|
||||
self.sheet_count += 1
|
||||
|
||||
def close(self) -> None:
|
||||
"""Finalize and write the workbook to disk."""
|
||||
if self.sheet_count == 0:
|
||||
# Excel requires at least one worksheet for a valid file.
|
||||
self._workbook.add_worksheet("Export")
|
||||
self._workbook.close()
|
||||
83
superset/utils/s3.py
Normal file
83
superset/utils/s3.py
Normal file
@@ -0,0 +1,83 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Minimal S3 helpers for uploading export artifacts and minting pre-signed URLs.
|
||||
|
||||
Credentials and region come from the standard boto3 resolution chain (env vars,
|
||||
shared config, instance role). Operators can override client construction via
|
||||
the ``EXCEL_EXPORT_S3_CLIENT_KWARGS`` config (e.g. ``region_name`` or an
|
||||
``endpoint_url`` for S3-compatible stores such as MinIO/LocalStack).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from flask import current_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_s3_client() -> Any:
|
||||
"""Build an S3 client using operator-provided client kwargs (if any)."""
|
||||
# boto3 is imported lazily so that importing this module (which happens at
|
||||
# app startup via the dashboard API) does not require boto3 to be installed.
|
||||
# The dependency is only needed when an export actually runs; if it is
|
||||
# missing, surface an actionable install hint rather than a bare ImportError.
|
||||
try:
|
||||
import boto3 # pylint: disable=import-outside-toplevel
|
||||
except ImportError as ex:
|
||||
raise ImportError(
|
||||
"boto3 is required for dashboard Excel export but is not installed. "
|
||||
"Install it with `pip install apache-superset[excel-export]`."
|
||||
) from ex
|
||||
|
||||
client_kwargs: dict[str, Any] = current_app.config.get(
|
||||
"EXCEL_EXPORT_S3_CLIENT_KWARGS", {}
|
||||
)
|
||||
return boto3.client("s3", **client_kwargs)
|
||||
|
||||
|
||||
def upload_file_to_s3(local_path: str, bucket: str, key: str) -> None:
|
||||
"""
|
||||
Upload a local file to S3.
|
||||
|
||||
``boto3``'s ``upload_file`` automatically uses a managed multipart transfer
|
||||
for large files, so no manual chunking is required.
|
||||
|
||||
:param local_path: Path to the file on local disk
|
||||
:param bucket: Destination S3 bucket
|
||||
:param key: Destination S3 object key
|
||||
"""
|
||||
_get_s3_client().upload_file(local_path, bucket, key)
|
||||
|
||||
|
||||
def generate_presigned_url(bucket: str, key: str, expires_in: int) -> str:
|
||||
"""
|
||||
Generate a time-limited pre-signed URL for downloading an S3 object.
|
||||
|
||||
:param bucket: The S3 bucket
|
||||
:param key: The S3 object key
|
||||
:param expires_in: URL lifetime in seconds
|
||||
:returns: A pre-signed ``get_object`` URL
|
||||
"""
|
||||
return _get_s3_client().generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": bucket, "Key": key},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
@@ -31,6 +31,7 @@ import yaml
|
||||
from freezegun import freeze_time
|
||||
from sqlalchemy import and_
|
||||
from superset import db, security_manager # noqa: F401
|
||||
from superset.exceptions import LockAlreadyHeldException
|
||||
from superset.models.dashboard import Dashboard
|
||||
from superset.models.core import FavStar, FavStarClassName
|
||||
from superset.reports.models import ReportSchedule, ReportScheduleType
|
||||
@@ -42,6 +43,7 @@ from superset.utils.core import backend, override_user
|
||||
from superset.utils.screenshots import ScreenshotCachePayload
|
||||
from superset.utils import json
|
||||
|
||||
from tests.conftest import with_config
|
||||
from tests.integration_tests.base_api_tests import ApiEditorsTestCaseMixin
|
||||
from tests.integration_tests.base_tests import (
|
||||
subjects_from_users,
|
||||
@@ -3262,6 +3264,173 @@ class TestDashboardApi(ApiEditorsTestCaseMixin, InsertChartMixin, SupersetTestCa
|
||||
response = json.loads(rv.data.decode("utf-8"))
|
||||
assert response["count"] > 0
|
||||
|
||||
def test_export_xlsx_501_when_bucket_unset(self):
|
||||
"""Dashboard API: export_xlsx returns 501 when the S3 bucket is unset."""
|
||||
admin = self.get_user("admin")
|
||||
dashboard = self.insert_dashboard("xlsx-501", None, [admin.id])
|
||||
self.login(ADMIN_USERNAME)
|
||||
try:
|
||||
rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/")
|
||||
assert rv.status_code == 501
|
||||
finally:
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_404_for_missing_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 404 for an unknown dashboard."""
|
||||
self.login(ADMIN_USERNAME)
|
||||
rv = self.client.post("api/v1/dashboard/99999999/export_xlsx/")
|
||||
assert rv.status_code == 404
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_400_for_empty_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 400 for a dashboard with no charts."""
|
||||
admin = self.get_user("admin")
|
||||
dashboard = self.insert_dashboard("xlsx-empty", None, [admin.id])
|
||||
self.login(ADMIN_USERNAME)
|
||||
try:
|
||||
rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/")
|
||||
assert rv.status_code == 400
|
||||
mock_task.apply_async.assert_not_called()
|
||||
finally:
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_202_enqueues_task(self, mock_task, mock_acquire):
|
||||
"""Dashboard API: export_xlsx enqueues the task and returns 202 + job_id."""
|
||||
self.login(ADMIN_USERNAME)
|
||||
dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first()
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}},
|
||||
)
|
||||
assert rv.status_code == 202
|
||||
body = json.loads(rv.data.decode("utf-8"))
|
||||
job_id = body["job_id"]
|
||||
assert job_id
|
||||
# The in-flight lock is acquired before the task is enqueued.
|
||||
mock_acquire.return_value.run.assert_called_once()
|
||||
mock_task.apply_async.assert_called_once()
|
||||
_, kwargs = mock_task.apply_async.call_args
|
||||
assert kwargs["task_id"] == job_id
|
||||
assert kwargs["kwargs"]["dashboard_id"] == dashboard.id
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_202_when_export_already_in_progress(
|
||||
self, mock_task, mock_acquire
|
||||
):
|
||||
"""Dashboard API: export_xlsx does not enqueue a second concurrent export."""
|
||||
# An in-flight lock is already held for this user+dashboard.
|
||||
mock_acquire.return_value.run.side_effect = LockAlreadyHeldException("held")
|
||||
self.login(ADMIN_USERNAME)
|
||||
dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first()
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}},
|
||||
)
|
||||
assert rv.status_code == 202
|
||||
assert "already in progress" in rv.data.decode("utf-8")
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_404_for_inaccessible_dashboard(self, mock_task):
|
||||
"""Dashboard API: export_xlsx returns 404 for a dashboard the user can't see."""
|
||||
admin = self.get_user("admin")
|
||||
dashboard = self.insert_dashboard(
|
||||
"xlsx-private", None, [admin.id], published=False
|
||||
)
|
||||
self.login(GAMMA_USERNAME)
|
||||
try:
|
||||
rv = self.client.post(f"api/v1/dashboard/{dashboard.id}/export_xlsx/")
|
||||
assert rv.status_code == 404
|
||||
mock_task.apply_async.assert_not_called()
|
||||
finally:
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
@patch("superset.dashboards.api.security_manager.raise_for_access")
|
||||
def test_export_xlsx_admitted_with_can_export_only(
|
||||
self, mock_raise, mock_task, mock_acquire
|
||||
):
|
||||
"""Dashboard API: export_xlsx is gated on ``can_export``, not a distinct
|
||||
``can_export_xlsx``. Gamma holds dashboard ``can_export`` by default (and
|
||||
the frontend shows the menu item on that basis), so a Gamma user must be
|
||||
admitted (202) rather than rejected by ``@protect()`` (403)."""
|
||||
gamma_user = security_manager.find_user(username=GAMMA_USERNAME)
|
||||
slice_ = db.session.query(Slice).first()
|
||||
# Clone Gamma (so the login password is valid); Gamma already carries
|
||||
# dashboard ``can_export``.
|
||||
with self.temporary_user(gamma_user, login=True) as user:
|
||||
dashboard = self.insert_dashboard(
|
||||
"xlsx-can-export", None, [user.id], slices=[slice_], published=True
|
||||
)
|
||||
try:
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}},
|
||||
)
|
||||
assert rv.status_code == 202
|
||||
mock_task.apply_async.assert_called_once()
|
||||
finally:
|
||||
db.session.delete(dashboard)
|
||||
db.session.commit()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_images_404_when_screenshot_flags_off(self, mock_task):
|
||||
"""Dashboard API: ``mode=images`` is rejected with 404 when the webdriver
|
||||
screenshot flags are disabled (the same signal the UI gates the option on),
|
||||
and no task is enqueued."""
|
||||
self.login(ADMIN_USERNAME)
|
||||
dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first()
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}, "mode": "images"},
|
||||
)
|
||||
assert rv.status_code == 404
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
@with_config({"EXCEL_EXPORT_S3_BUCKET": "exports"})
|
||||
@with_feature_flags(
|
||||
ENABLE_DASHBOARD_SCREENSHOT_ENDPOINTS=True,
|
||||
ENABLE_DASHBOARD_DOWNLOAD_WEBDRIVER_SCREENSHOT=True,
|
||||
)
|
||||
@patch("superset.dashboards.api.AcquireDistributedLock")
|
||||
@patch("superset.dashboards.api.export_dashboard_excel")
|
||||
def test_export_xlsx_images_202_when_screenshot_flags_on(
|
||||
self, mock_task, mock_acquire
|
||||
):
|
||||
"""Dashboard API: ``mode=images`` is accepted (202) when both webdriver
|
||||
screenshot flags are enabled."""
|
||||
self.login(ADMIN_USERNAME)
|
||||
dashboard = db.session.query(Dashboard).filter_by(slug="world_health").first()
|
||||
rv = self.client.post(
|
||||
f"api/v1/dashboard/{dashboard.id}/export_xlsx/",
|
||||
json={"active_data_mask": {}, "mode": "images"},
|
||||
)
|
||||
assert rv.status_code == 202
|
||||
mock_task.apply_async.assert_called_once()
|
||||
_, kwargs = mock_task.apply_async.call_args
|
||||
assert kwargs["kwargs"]["mode"] == "images"
|
||||
|
||||
@pytest.mark.usefixtures("load_world_bank_dashboard_with_slices")
|
||||
def test_embedded_dashboards(self):
|
||||
self.login(ADMIN_USERNAME)
|
||||
|
||||
@@ -563,6 +563,178 @@ def test_get_dashboard_filter_context_out_of_scope_filter_excluded(
|
||||
assert ctx.filters[0].id == "f1"
|
||||
|
||||
|
||||
def _build_dashboard_mock(
|
||||
mock_db: MagicMock,
|
||||
filter_config: list[dict[str, Any]],
|
||||
chart_ids: list[int],
|
||||
) -> MagicMock:
|
||||
"""Wire a dashboard MagicMock with the given filters and chart ids."""
|
||||
metadata = {"native_filter_configuration": filter_config}
|
||||
dashboard = MagicMock()
|
||||
dashboard.id = 1
|
||||
dashboard.slices = [MagicMock(id=cid) for cid in chart_ids]
|
||||
dashboard.json_metadata = json.dumps(metadata)
|
||||
dashboard.position_json = json.dumps(SAMPLE_POSITION_JSON)
|
||||
(
|
||||
mock_db.session.query.return_value.filter_by.return_value.one_or_none.return_value
|
||||
) = dashboard
|
||||
return dashboard
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_active_data_mask_overrides_default(
|
||||
mock_db: MagicMock,
|
||||
mock_check_access: MagicMock,
|
||||
) -> None:
|
||||
"""An active filter value replaces the saved default."""
|
||||
filter_config = [
|
||||
_make_filter(
|
||||
flt_id="f1",
|
||||
name="Region",
|
||||
scope_root=["ROOT_ID"],
|
||||
default_value=["US"],
|
||||
target_column="region",
|
||||
),
|
||||
]
|
||||
_build_dashboard_mock(mock_db, filter_config, [10])
|
||||
|
||||
active_data_mask = {
|
||||
"f1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "region", "op": "IN", "val": ["FR", "DE"]}]
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx = get_dashboard_filter_context(
|
||||
dashboard_id=1, chart_id=10, active_data_mask=active_data_mask
|
||||
)
|
||||
|
||||
assert ctx.filters[0].status == DashboardFilterStatus.APPLIED
|
||||
assert ctx.extra_form_data["filters"][0]["val"] == ["FR", "DE"]
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_active_data_mask_empty_clears_default(
|
||||
mock_db: MagicMock,
|
||||
mock_check_access: MagicMock,
|
||||
) -> None:
|
||||
"""An empty active extraFormData clears the filter; the default is NOT used."""
|
||||
filter_config = [
|
||||
_make_filter(
|
||||
flt_id="f1",
|
||||
name="Region",
|
||||
scope_root=["ROOT_ID"],
|
||||
default_value=["US"],
|
||||
target_column="region",
|
||||
),
|
||||
]
|
||||
_build_dashboard_mock(mock_db, filter_config, [10])
|
||||
|
||||
active_data_mask: dict[str, Any] = {"f1": {"extraFormData": {}}}
|
||||
ctx = get_dashboard_filter_context(
|
||||
dashboard_id=1, chart_id=10, active_data_mask=active_data_mask
|
||||
)
|
||||
|
||||
assert ctx.filters[0].status == DashboardFilterStatus.NOT_APPLIED
|
||||
assert "filters" not in ctx.extra_form_data
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_active_data_mask_absent_filter_falls_back_to_default(
|
||||
mock_db: MagicMock,
|
||||
mock_check_access: MagicMock,
|
||||
) -> None:
|
||||
"""A filter not present in the mask keeps its saved default."""
|
||||
filter_config = [
|
||||
_make_filter(
|
||||
flt_id="f1",
|
||||
name="Region",
|
||||
scope_root=["ROOT_ID"],
|
||||
default_value=["US"],
|
||||
target_column="region",
|
||||
),
|
||||
]
|
||||
_build_dashboard_mock(mock_db, filter_config, [10])
|
||||
|
||||
# Mask only references some other filter id
|
||||
active_data_mask: dict[str, Any] = {"f2": {"extraFormData": {"filters": []}}}
|
||||
ctx = get_dashboard_filter_context(
|
||||
dashboard_id=1, chart_id=10, active_data_mask=active_data_mask
|
||||
)
|
||||
|
||||
assert ctx.filters[0].status == DashboardFilterStatus.APPLIED
|
||||
assert ctx.extra_form_data["filters"][0]["val"] == ["US"]
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_active_data_mask_applies_despite_default_to_first_item(
|
||||
mock_db: MagicMock,
|
||||
mock_check_access: MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
defaultToFirstItem filters cannot be resolved from saved config, but when the
|
||||
frontend supplies a concrete active value it is applied.
|
||||
"""
|
||||
filter_config = [
|
||||
_make_filter(
|
||||
flt_id="f1",
|
||||
name="City",
|
||||
scope_root=["ROOT_ID"],
|
||||
default_to_first_item=True,
|
||||
target_column="city",
|
||||
),
|
||||
]
|
||||
_build_dashboard_mock(mock_db, filter_config, [10])
|
||||
|
||||
active_data_mask = {
|
||||
"f1": {
|
||||
"extraFormData": {"filters": [{"col": "city", "op": "IN", "val": ["NYC"]}]}
|
||||
}
|
||||
}
|
||||
ctx = get_dashboard_filter_context(
|
||||
dashboard_id=1, chart_id=10, active_data_mask=active_data_mask
|
||||
)
|
||||
|
||||
assert ctx.filters[0].status == DashboardFilterStatus.APPLIED
|
||||
assert ctx.extra_form_data["filters"][0]["val"] == ["NYC"]
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_active_data_mask_out_of_scope_filter_still_excluded(
|
||||
mock_db: MagicMock,
|
||||
mock_check_access: MagicMock,
|
||||
) -> None:
|
||||
"""An active value for an out-of-scope filter does not leak into the chart."""
|
||||
filter_config = [
|
||||
_make_filter(
|
||||
flt_id="f1",
|
||||
name="Out-of-scope",
|
||||
scope_root=["TABS-nonexistent"],
|
||||
target_column="status",
|
||||
),
|
||||
]
|
||||
_build_dashboard_mock(mock_db, filter_config, [10])
|
||||
|
||||
active_data_mask = {
|
||||
"f1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "status", "op": "IN", "val": ["active"]}]
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx = get_dashboard_filter_context(
|
||||
dashboard_id=1, chart_id=10, active_data_mask=active_data_mask
|
||||
)
|
||||
|
||||
assert ctx.filters == []
|
||||
assert ctx.extra_form_data == {}
|
||||
|
||||
|
||||
@patch("superset.charts.data.dashboard_filter_context._check_dashboard_access")
|
||||
@patch("superset.charts.data.dashboard_filter_context.db")
|
||||
def test_get_dashboard_filter_context_chart_not_in_layout_receives_root_filters(
|
||||
|
||||
157
tests/unit_tests/dashboards/test_excel_export_email.py
Normal file
157
tests/unit_tests/dashboards/test_excel_export_email.py
Normal file
@@ -0,0 +1,157 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from superset.dashboards.excel_export import email
|
||||
|
||||
REQUESTED = datetime(2026, 1, 1, 12, 0, 0)
|
||||
EXPIRES = datetime(2026, 1, 2, 12, 0, 0)
|
||||
|
||||
|
||||
def test_success_email_contains_link_and_expiry() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://signed.example/file.xlsx?sig=abc",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=86400,
|
||||
errored={},
|
||||
)
|
||||
assert "https://signed.example/file.xlsx?sig=abc" in html
|
||||
assert "expires in 24 hours" in html
|
||||
assert "2026-01-02 12:00:00 UTC" in html
|
||||
assert "2026-01-01 12:00:00 UTC" in html
|
||||
assert "<li>" not in html # no skipped section
|
||||
|
||||
|
||||
def test_success_email_sub_hour_ttl_reports_minutes() -> None:
|
||||
# A sub-hour TTL must not truncate to "0 hours"; it should report minutes.
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=900,
|
||||
errored={},
|
||||
)
|
||||
assert "expires in 15 minutes" in html
|
||||
assert "0 hours" not in html
|
||||
|
||||
|
||||
def test_success_email_mixed_ttl_reports_hours_and_minutes() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=5400,
|
||||
errored={},
|
||||
)
|
||||
assert "expires in 1 hour 30 minutes" in html
|
||||
|
||||
|
||||
def test_success_email_lists_charts_with_no_query_context() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=86400,
|
||||
errored={email.ERROR_NO_QUERY_CONTEXT: ["10 - Broken chart"]},
|
||||
)
|
||||
assert "no saved query context" in html
|
||||
assert "<li>10 - Broken chart</li>" in html
|
||||
|
||||
|
||||
def test_success_email_groups_errors_by_reason() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=86400,
|
||||
errored={
|
||||
email.ERROR_NO_QUERY_CONTEXT: ["10 - NoContext"],
|
||||
email.ERROR_GENERAL: ["30 - Boom"],
|
||||
},
|
||||
)
|
||||
# Each reason renders its own labelled section with the right chart.
|
||||
assert "no saved query context" in html
|
||||
assert "an error occurred" in html
|
||||
assert "<li>10 - NoContext</li>" in html
|
||||
assert "<li>30 - Boom</li>" in html
|
||||
|
||||
|
||||
def test_success_email_omits_empty_reason_groups() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="Sales",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=86400,
|
||||
errored={email.ERROR_GENERAL: ["30 - Boom"]},
|
||||
)
|
||||
assert "an error occurred" in html
|
||||
assert "no saved query context" not in html
|
||||
assert "<li>30 - Boom</li>" in html
|
||||
|
||||
|
||||
def test_success_email_escapes_title() -> None:
|
||||
html = email.build_success_email(
|
||||
dashboard_title="<script>alert(1)</script>",
|
||||
download_url="https://x",
|
||||
requested_at=REQUESTED,
|
||||
expires_at=EXPIRES,
|
||||
ttl_seconds=86400,
|
||||
errored={},
|
||||
)
|
||||
assert "<script>" not in html
|
||||
assert "<script>" in html
|
||||
|
||||
|
||||
def test_failure_email_body() -> None:
|
||||
html = email.build_failure_email("Sales", REQUESTED)
|
||||
assert "could not be completed" in html
|
||||
assert "2026-01-01 12:00:00 UTC" in html
|
||||
|
||||
|
||||
@patch("superset.dashboards.excel_export.email.current_app")
|
||||
def test_build_subject(mock_app: MagicMock) -> None:
|
||||
mock_app.config = {"EMAIL_REPORTS_SUBJECT_PREFIX": "[Report] "}
|
||||
assert (
|
||||
email.build_subject("Sales", success=True)
|
||||
== "[Report] Your dashboard export is ready: Sales"
|
||||
)
|
||||
assert email.build_subject("Sales", success=False).startswith(
|
||||
"[Report] Your dashboard export could not be completed"
|
||||
)
|
||||
|
||||
|
||||
@patch("superset.dashboards.excel_export.email.send_email_smtp")
|
||||
@patch("superset.dashboards.excel_export.email.current_app")
|
||||
def test_send_export_email(mock_app: MagicMock, mock_send: MagicMock) -> None:
|
||||
mock_app.config = {"SMTP_HOST": "localhost"}
|
||||
email.send_export_email("user@example.com", "subj", "<html></html>")
|
||||
mock_send.assert_called_once_with(
|
||||
to="user@example.com",
|
||||
subject="subj",
|
||||
html_content="<html></html>",
|
||||
config=mock_app.config,
|
||||
)
|
||||
105
tests/unit_tests/dashboards/test_excel_export_layout.py
Normal file
105
tests/unit_tests/dashboards/test_excel_export_layout.py
Normal file
@@ -0,0 +1,105 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from superset.dashboards.excel_export.layout import get_charts_in_layout_order
|
||||
|
||||
|
||||
def _chart_node(node_id: str, chart_id: int) -> dict[str, Any]:
|
||||
return {"id": node_id, "type": "CHART", "meta": {"chartId": chart_id}}
|
||||
|
||||
|
||||
def _dashboard(position: dict[str, Any], chart_ids: list[int]) -> MagicMock:
|
||||
dashboard = MagicMock()
|
||||
dashboard.position = position
|
||||
dashboard.slices = [MagicMock(id=cid) for cid in chart_ids]
|
||||
return dashboard
|
||||
|
||||
|
||||
def _ids(slices: list[Any]) -> list[int]:
|
||||
return [slc.id for slc in slices]
|
||||
|
||||
|
||||
def test_grid_order() -> None:
|
||||
position = {
|
||||
"ROOT_ID": {"type": "ROOT", "children": ["GRID_ID"]},
|
||||
"GRID_ID": {"type": "GRID", "children": ["ROW-1"]},
|
||||
"ROW-1": {"type": "ROW", "children": ["CHART-a", "CHART-b"]},
|
||||
"CHART-a": _chart_node("CHART-a", 1),
|
||||
"CHART-b": _chart_node("CHART-b", 2),
|
||||
}
|
||||
dashboard = _dashboard(position, [2, 1])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [1, 2]
|
||||
|
||||
|
||||
def test_tab_nested_order() -> None:
|
||||
position = {
|
||||
"ROOT_ID": {"type": "ROOT", "children": ["GRID_ID"]},
|
||||
"GRID_ID": {"type": "GRID", "children": ["TABS-1"]},
|
||||
"TABS-1": {"type": "TABS", "children": ["TAB-1", "TAB-2"]},
|
||||
"TAB-1": {"type": "TAB", "children": ["CHART-a"]},
|
||||
"TAB-2": {"type": "TAB", "children": ["CHART-b"]},
|
||||
"CHART-a": _chart_node("CHART-a", 10),
|
||||
"CHART-b": _chart_node("CHART-b", 20),
|
||||
}
|
||||
dashboard = _dashboard(position, [20, 10])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [10, 20]
|
||||
|
||||
|
||||
def test_duplicate_chart_placement_exported_once() -> None:
|
||||
position = {
|
||||
"ROOT_ID": {"type": "ROOT", "children": ["GRID_ID"]},
|
||||
"GRID_ID": {"type": "GRID", "children": ["ROW-1", "ROW-2"]},
|
||||
"ROW-1": {"type": "ROW", "children": ["CHART-a"]},
|
||||
"ROW-2": {"type": "ROW", "children": ["CHART-a-dup"]},
|
||||
"CHART-a": _chart_node("CHART-a", 5),
|
||||
"CHART-a-dup": _chart_node("CHART-a-dup", 5),
|
||||
}
|
||||
dashboard = _dashboard(position, [5])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [5]
|
||||
|
||||
|
||||
def test_orphan_charts_appended_by_id() -> None:
|
||||
position = {
|
||||
"ROOT_ID": {"type": "ROOT", "children": ["GRID_ID"]},
|
||||
"GRID_ID": {"type": "GRID", "children": ["ROW-1"]},
|
||||
"ROW-1": {"type": "ROW", "children": ["CHART-a"]},
|
||||
"CHART-a": _chart_node("CHART-a", 7),
|
||||
}
|
||||
# Charts 3 and 9 are on the dashboard but not in the layout.
|
||||
dashboard = _dashboard(position, [7, 9, 3])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [7, 3, 9]
|
||||
|
||||
|
||||
def test_stale_layout_chart_skipped() -> None:
|
||||
position = {
|
||||
"ROOT_ID": {"type": "ROOT", "children": ["GRID_ID"]},
|
||||
"GRID_ID": {"type": "GRID", "children": ["ROW-1"]},
|
||||
"ROW-1": {"type": "ROW", "children": ["CHART-a", "CHART-gone"]},
|
||||
"CHART-a": _chart_node("CHART-a", 1),
|
||||
"CHART-gone": _chart_node("CHART-gone", 999), # not in dashboard.slices
|
||||
}
|
||||
dashboard = _dashboard(position, [1])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [1]
|
||||
|
||||
|
||||
def test_empty_position_returns_all_slices_sorted() -> None:
|
||||
dashboard = _dashboard({}, [3, 1, 2])
|
||||
assert _ids(get_charts_in_layout_order(dashboard)) == [1, 2, 3]
|
||||
164
tests/unit_tests/dashboards/test_excel_export_screenshot.py
Normal file
164
tests/unit_tests/dashboards/test_excel_export_screenshot.py
Normal file
@@ -0,0 +1,164 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
|
||||
from superset.charts.data.dashboard_filter_context import DashboardFilterContext
|
||||
from superset.dashboards.excel_export import screenshot as screenshot_module
|
||||
from superset.dashboards.excel_export.screenshot import render_chart_image
|
||||
from superset.utils import json
|
||||
|
||||
MODULE = "superset.dashboards.excel_export.screenshot"
|
||||
|
||||
|
||||
def _chart(
|
||||
chart_id: int = 42,
|
||||
params: str = '{"viz_type": "pie"}',
|
||||
digest: str = "abc123",
|
||||
) -> MagicMock:
|
||||
chart = MagicMock()
|
||||
chart.id = chart_id
|
||||
chart.params = params
|
||||
chart.digest = digest
|
||||
return chart
|
||||
|
||||
|
||||
def _patched_app() -> Any:
|
||||
app = MagicMock()
|
||||
app.config = {"WEBDRIVER_WINDOW": {"slice": (3000, 1200)}}
|
||||
return app
|
||||
|
||||
|
||||
def test_render_chart_image_builds_url_with_slice_id_and_filters() -> None:
|
||||
chart = _chart()
|
||||
active_mask = {"NATIVE_FILTER-1": {"extraFormData": {"filters": [{"col": "a"}]}}}
|
||||
filter_context = DashboardFilterContext(
|
||||
extra_form_data={"filters": [{"col": "a", "op": "IN", "val": ["x"]}]}
|
||||
)
|
||||
|
||||
screenshot_instance = MagicMock()
|
||||
screenshot_instance.get_screenshot.return_value = b"PNGBYTES"
|
||||
|
||||
with (
|
||||
patch(
|
||||
f"{MODULE}.get_dashboard_filter_context", return_value=filter_context
|
||||
) as mock_ctx,
|
||||
patch(f"{MODULE}.get_url_path", return_value="/explore/url") as mock_url,
|
||||
patch(
|
||||
f"{MODULE}.ChartScreenshot", return_value=screenshot_instance
|
||||
) as mock_screenshot,
|
||||
patch.object(screenshot_module, "current_app", _patched_app()),
|
||||
):
|
||||
user = MagicMock()
|
||||
result = render_chart_image(
|
||||
chart, dashboard_id=7, active_data_mask=active_mask, user=user
|
||||
)
|
||||
|
||||
assert result == b"PNGBYTES"
|
||||
|
||||
# Live filter state is resolved for this chart on this dashboard.
|
||||
mock_ctx.assert_called_once_with(
|
||||
dashboard_id=7, chart_id=42, active_data_mask=active_mask
|
||||
)
|
||||
|
||||
# The Explore URL carries the slice id and the live extra_form_data.
|
||||
_, url_kwargs = mock_url.call_args
|
||||
form_data = json.loads(url_kwargs["form_data"])
|
||||
assert form_data["slice_id"] == 42
|
||||
assert form_data["viz_type"] == "pie"
|
||||
assert form_data["extra_form_data"] == filter_context.extra_form_data
|
||||
|
||||
# ChartScreenshot is built with the chart digest + slice window sizing.
|
||||
args, kwargs = mock_screenshot.call_args
|
||||
assert args[0] == "/explore/url"
|
||||
assert args[1] == "abc123"
|
||||
assert kwargs["window_size"] == (3000, 1200)
|
||||
assert kwargs["thumb_size"] == (3000, 1200)
|
||||
screenshot_instance.get_screenshot.assert_called_once_with(user=user)
|
||||
|
||||
|
||||
def test_render_chart_image_omits_extra_form_data_when_no_filters() -> None:
|
||||
chart = _chart()
|
||||
filter_context = DashboardFilterContext(extra_form_data={})
|
||||
screenshot_instance = MagicMock()
|
||||
screenshot_instance.get_screenshot.return_value = b"PNG"
|
||||
|
||||
with (
|
||||
patch(f"{MODULE}.get_dashboard_filter_context", return_value=filter_context),
|
||||
patch(f"{MODULE}.get_url_path", return_value="/u") as mock_url,
|
||||
patch(f"{MODULE}.ChartScreenshot", return_value=screenshot_instance),
|
||||
patch.object(screenshot_module, "current_app", _patched_app()),
|
||||
):
|
||||
render_chart_image(chart, dashboard_id=7, active_data_mask={}, user=MagicMock())
|
||||
|
||||
_, url_kwargs = mock_url.call_args
|
||||
form_data = json.loads(url_kwargs["form_data"])
|
||||
assert "extra_form_data" not in form_data
|
||||
|
||||
|
||||
def test_render_chart_image_returns_none_when_screenshot_fails() -> None:
|
||||
chart = _chart()
|
||||
screenshot_instance = MagicMock()
|
||||
screenshot_instance.get_screenshot.return_value = None
|
||||
|
||||
with (
|
||||
patch(
|
||||
f"{MODULE}.get_dashboard_filter_context",
|
||||
return_value=DashboardFilterContext(),
|
||||
),
|
||||
patch(f"{MODULE}.get_url_path", return_value="/u"),
|
||||
patch(f"{MODULE}.ChartScreenshot", return_value=screenshot_instance),
|
||||
patch.object(screenshot_module, "current_app", _patched_app()),
|
||||
):
|
||||
result = render_chart_image(
|
||||
chart, dashboard_id=7, active_data_mask={}, user=MagicMock()
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_render_chart_image_returns_none_on_exception() -> None:
|
||||
chart = _chart()
|
||||
|
||||
with patch(
|
||||
f"{MODULE}.get_dashboard_filter_context", side_effect=ValueError("boom")
|
||||
):
|
||||
result = render_chart_image(
|
||||
chart, dashboard_id=7, active_data_mask={}, user=MagicMock()
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_render_chart_image_propagates_soft_time_limit() -> None:
|
||||
# A soft timeout must abort the export, not be swallowed into a ``None``
|
||||
# result (which the caller would mis-report as a per-chart render failure).
|
||||
chart = _chart()
|
||||
|
||||
with (
|
||||
patch(
|
||||
f"{MODULE}.get_dashboard_filter_context",
|
||||
side_effect=SoftTimeLimitExceeded(),
|
||||
),
|
||||
pytest.raises(SoftTimeLimitExceeded),
|
||||
):
|
||||
render_chart_image(chart, dashboard_id=7, active_data_mask={}, user=MagicMock())
|
||||
401
tests/unit_tests/tasks/test_export_dashboard_excel.py
Normal file
401
tests/unit_tests/tasks/test_export_dashboard_excel.py
Normal file
@@ -0,0 +1,401 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
|
||||
from superset.utils import json
|
||||
|
||||
MODULE = "superset.tasks.export_dashboard_excel"
|
||||
|
||||
|
||||
# A minimal valid 1x1 transparent PNG for image-mode tests.
|
||||
_PNG_1x1 = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
|
||||
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc\x00\x01\x00\x00\x05\x00"
|
||||
b"\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
|
||||
|
||||
def _chart(
|
||||
chart_id: int,
|
||||
name: str,
|
||||
has_context: bool = True,
|
||||
viz_type: str = "line",
|
||||
) -> mock.MagicMock:
|
||||
chart = mock.MagicMock()
|
||||
chart.id = chart_id
|
||||
chart.slice_name = name
|
||||
chart.viz_type = viz_type
|
||||
chart.query_context = json.dumps({"queries": [{}]}) if has_context else None
|
||||
return chart
|
||||
|
||||
|
||||
def _media(path: str) -> list[str]:
|
||||
"""Embedded media entries of an xlsx (which is a zip archive)."""
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
return [n for n in archive.namelist() if n.startswith("xl/media/")]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocks() -> Iterator[dict[str, Any]]:
|
||||
"""Patch every external dependency of the task; keep the real xlsx writer."""
|
||||
with ExitStack() as stack:
|
||||
# Use explicit MagicMock instances: patch() auto-creates async-flavored
|
||||
# mocks for these targets (their real objects expose async members), which
|
||||
# would make calls like security_manager.get_user_by_id() return coroutines.
|
||||
patched = {
|
||||
name: stack.enter_context(
|
||||
mock.patch(f"{MODULE}.{name}", new=mock.MagicMock())
|
||||
)
|
||||
for name in (
|
||||
"security_manager",
|
||||
"db",
|
||||
"get_charts_in_layout_order",
|
||||
"get_dashboard_filter_context",
|
||||
"ChartDataQueryContextSchema",
|
||||
"ChartDataCommand",
|
||||
"render_chart_image",
|
||||
"s3",
|
||||
"email",
|
||||
"ReleaseDistributedLock",
|
||||
)
|
||||
}
|
||||
user = mock.MagicMock()
|
||||
user.email = "user@example.com"
|
||||
patched["security_manager"].get_user_by_id.return_value = user
|
||||
|
||||
dashboard = mock.MagicMock()
|
||||
dashboard.id = 1
|
||||
dashboard.dashboard_title = "Sales"
|
||||
patched[
|
||||
"db"
|
||||
].session.query.return_value.filter_by.return_value.one_or_none.return_value = ( # noqa: E501
|
||||
dashboard
|
||||
)
|
||||
|
||||
patched["get_dashboard_filter_context"].return_value.extra_form_data = {}
|
||||
patched["s3"].generate_presigned_url.return_value = "https://signed/file.xlsx"
|
||||
|
||||
patched["user"] = user
|
||||
patched["dashboard"] = dashboard
|
||||
yield patched
|
||||
|
||||
|
||||
def _run(
|
||||
job_id: str = "job-1",
|
||||
mode: str = "data",
|
||||
) -> None:
|
||||
from superset.tasks.export_dashboard_excel import export_dashboard_excel
|
||||
|
||||
export_dashboard_excel(
|
||||
dashboard_id=1,
|
||||
user_id=2,
|
||||
active_data_mask={},
|
||||
job_id=job_id,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def _no_temp_files_left(job_id: str) -> bool:
|
||||
pattern = os.path.join(tempfile.gettempdir(), f"dash-export-{job_id}-*")
|
||||
return glob.glob(pattern) == []
|
||||
|
||||
|
||||
def _read_sheets(path: str) -> dict[str, list[list[object]]]:
|
||||
openpyxl = pytest.importorskip("openpyxl")
|
||||
workbook = openpyxl.load_workbook(path, read_only=True)
|
||||
sheets = {
|
||||
ws.title: [list(r) for r in ws.iter_rows(values_only=True)]
|
||||
for ws in workbook.worksheets
|
||||
}
|
||||
workbook.close()
|
||||
return sheets
|
||||
|
||||
|
||||
def test_happy_path_uploads_and_emails(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "First"),
|
||||
_chart(20, "Second"),
|
||||
]
|
||||
mocks["ChartDataCommand"].return_value.run.side_effect = [
|
||||
{"queries": [{"colnames": ["a", "b"], "data": [{"a": 1, "b": 2}]}]},
|
||||
{"queries": [{"colnames": ["c"], "data": [{"c": "x"}]}]},
|
||||
]
|
||||
|
||||
# Capture the workbook before the task deletes it.
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
def _capture(path: str, bucket: str, key: str) -> None:
|
||||
uploaded["sheets"] = _read_sheets(path)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.side_effect = _capture
|
||||
|
||||
_run()
|
||||
|
||||
mocks["s3"].upload_file_to_s3.assert_called_once()
|
||||
assert list(uploaded["sheets"].keys()) == ["10 - First", "20 - Second"]
|
||||
mocks["email"].send_export_email.assert_called_once()
|
||||
mocks["email"].build_success_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-1")
|
||||
|
||||
|
||||
def test_chart_without_query_context_is_skipped(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Good"),
|
||||
_chart(20, "NoContext", has_context=False),
|
||||
]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {
|
||||
mocks["email"].ERROR_NO_QUERY_CONTEXT: ["20 - NoContext"]
|
||||
}
|
||||
|
||||
|
||||
def test_chart_query_error_grouped_as_general_export_continues(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Boom"),
|
||||
_chart(20, "Ok"),
|
||||
]
|
||||
mocks["ChartDataCommand"].return_value.run.side_effect = [
|
||||
RuntimeError("query failed"),
|
||||
{"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]},
|
||||
]
|
||||
|
||||
_run()
|
||||
|
||||
mocks["s3"].upload_file_to_s3.assert_called_once()
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {mocks["email"].ERROR_GENERAL: ["10 - Boom"]}
|
||||
|
||||
|
||||
def test_chart_timeout_aborts_export_and_sends_failure_email(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
# A soft timeout raised while a chart runs must abort the whole export
|
||||
# (propagate to the outer handler) rather than being recorded per-chart and
|
||||
# letting the task run on until the hard limit kills the worker — which would
|
||||
# skip cleanup, leak temp files, and never send a failure email.
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Ok"),
|
||||
_chart(20, "Slow"),
|
||||
]
|
||||
mocks["ChartDataCommand"].return_value.run.side_effect = [
|
||||
{"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]},
|
||||
SoftTimeLimitExceeded(),
|
||||
]
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-timeout")
|
||||
|
||||
mocks["s3"].upload_file_to_s3.assert_not_called()
|
||||
mocks["email"].build_success_email.assert_not_called()
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-timeout")
|
||||
|
||||
|
||||
def test_image_render_timeout_aborts_export(mocks: dict[str, Any]) -> None:
|
||||
# In image mode a soft timeout during a chart render must also propagate and
|
||||
# abort the export rather than being swallowed as a per-chart render failure.
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Line", viz_type="line"),
|
||||
]
|
||||
mocks["render_chart_image"].side_effect = SoftTimeLimitExceeded()
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-img-timeout", mode="images")
|
||||
|
||||
mocks["email"].build_success_email.assert_not_called()
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-img-timeout")
|
||||
|
||||
|
||||
def test_all_charts_skipped_writes_summary(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "NoContext", has_context=False),
|
||||
]
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
def _capture(path: str, bucket: str, key: str) -> None:
|
||||
uploaded["sheets"] = _read_sheets(path)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.side_effect = _capture
|
||||
|
||||
_run()
|
||||
|
||||
assert "Export Summary" in uploaded["sheets"]
|
||||
mocks["email"].build_success_email.assert_called_once()
|
||||
|
||||
|
||||
def test_upload_failure_sends_failure_email_and_cleans_up(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
mocks["s3"].upload_file_to_s3.side_effect = RuntimeError("s3 down")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
_run("job-fail")
|
||||
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
mocks["email"].send_export_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-fail")
|
||||
|
||||
|
||||
def test_soft_time_limit_sends_failure_email(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].side_effect = SoftTimeLimitExceeded()
|
||||
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
_run("job-timeout")
|
||||
|
||||
mocks["email"].build_failure_email.assert_called_once()
|
||||
assert _no_temp_files_left("job-timeout")
|
||||
|
||||
|
||||
# --- image mode ---
|
||||
|
||||
|
||||
def test_images_mode_embeds_non_table_and_keeps_tables_tabular(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Line", viz_type="line"),
|
||||
_chart(20, "Tbl", viz_type="table"),
|
||||
]
|
||||
mocks["render_chart_image"].return_value = _PNG_1x1
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
def _capture(path: str, bucket: str, key: str) -> None:
|
||||
uploaded["sheets"] = _read_sheets(path)
|
||||
uploaded["media"] = _media(path)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.side_effect = _capture
|
||||
|
||||
_run(mode="images")
|
||||
|
||||
# The non-table chart is rendered as an image (with the requesting user)...
|
||||
mocks["render_chart_image"].assert_called_once()
|
||||
render_args = mocks["render_chart_image"].call_args[0]
|
||||
assert render_args[0].id == 10
|
||||
assert render_args[3] is mocks["user"]
|
||||
# ...and the table chart still goes through the data path.
|
||||
mocks["ChartDataCommand"].return_value.run.assert_called_once()
|
||||
|
||||
assert set(uploaded["sheets"].keys()) == {"10 - Line", "20 - Tbl"}
|
||||
# Exactly one embedded image (the non-table chart).
|
||||
assert len(uploaded["media"]) == 1
|
||||
|
||||
|
||||
def test_images_mode_renders_chart_without_query_context(
|
||||
mocks: dict[str, Any],
|
||||
) -> None:
|
||||
# An image chart with no saved query context can still render.
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Line", viz_type="line", has_context=False),
|
||||
]
|
||||
mocks["render_chart_image"].return_value = _PNG_1x1
|
||||
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
def _capture(path: str, bucket: str, key: str) -> None:
|
||||
uploaded["media"] = _media(path)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.side_effect = _capture
|
||||
|
||||
_run(mode="images")
|
||||
|
||||
mocks["render_chart_image"].assert_called_once()
|
||||
assert len(uploaded["media"]) == 1
|
||||
|
||||
|
||||
def test_images_mode_none_render_is_skipped(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [
|
||||
_chart(10, "Line", viz_type="line"),
|
||||
]
|
||||
mocks["render_chart_image"].return_value = None
|
||||
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
def _capture(path: str, bucket: str, key: str) -> None:
|
||||
uploaded["sheets"] = _read_sheets(path)
|
||||
|
||||
mocks["s3"].upload_file_to_s3.side_effect = _capture
|
||||
|
||||
_run(mode="images")
|
||||
|
||||
# A chart that cannot render is grouped under the general-error reason.
|
||||
_, kwargs = mocks["email"].build_success_email.call_args
|
||||
assert kwargs["errored"] == {mocks["email"].ERROR_GENERAL: ["10 - Line"]}
|
||||
# Nothing rendered → the summary sheet stands in for an empty workbook.
|
||||
assert "Export Summary" in uploaded["sheets"]
|
||||
|
||||
|
||||
def test_inflight_lock_released_on_success(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
|
||||
_run()
|
||||
|
||||
# The distributed lock is released for this user+dashboard when the task
|
||||
# settles (namespace + params match what the API acquired).
|
||||
mocks["ReleaseDistributedLock"].assert_called_once_with(
|
||||
"excel_export", {"user_id": 2, "dashboard_id": 1}
|
||||
)
|
||||
mocks["ReleaseDistributedLock"].return_value.run.assert_called_once_with()
|
||||
|
||||
|
||||
def test_inflight_lock_released_on_failure(mocks: dict[str, Any]) -> None:
|
||||
mocks["get_charts_in_layout_order"].return_value = [_chart(10, "Good")]
|
||||
mocks["ChartDataCommand"].return_value.run.return_value = {
|
||||
"queries": [{"colnames": ["a"], "data": [{"a": 1}]}]
|
||||
}
|
||||
mocks["s3"].upload_file_to_s3.side_effect = RuntimeError("s3 down")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
_run("job-fail")
|
||||
|
||||
# The lock is freed in ``finally`` even when the export fails.
|
||||
mocks["ReleaseDistributedLock"].assert_called_once_with(
|
||||
"excel_export", {"user_id": 2, "dashboard_id": 1}
|
||||
)
|
||||
mocks["ReleaseDistributedLock"].return_value.run.assert_called_once_with()
|
||||
267
tests/unit_tests/utils/excel_streaming_tests.py
Normal file
267
tests/unit_tests/utils/excel_streaming_tests.py
Normal file
@@ -0,0 +1,267 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils import excel_streaming
|
||||
from superset.utils.excel_streaming import (
|
||||
_sanitize_cell,
|
||||
sanitize_sheet_name,
|
||||
StreamingXlsxWriter,
|
||||
)
|
||||
|
||||
# --- sanitize_sheet_name ---
|
||||
|
||||
|
||||
def test_sheet_name_replaces_forbidden_chars() -> None:
|
||||
assert sanitize_sheet_name("a/b:c*d?e[f]g\\h", set()) == "a_b_c_d_e_f_g_h"
|
||||
|
||||
|
||||
def test_sheet_name_truncated_to_31() -> None:
|
||||
assert sanitize_sheet_name("x" * 40, set()) == "x" * 31
|
||||
|
||||
|
||||
def test_sheet_name_dedupes_case_insensitively() -> None:
|
||||
used: set[str] = set()
|
||||
assert sanitize_sheet_name("Sales", used) == "Sales"
|
||||
assert sanitize_sheet_name("sales", used) == "sales~2"
|
||||
assert sanitize_sheet_name("SALES", used) == "SALES~3"
|
||||
|
||||
|
||||
def test_sheet_name_dedupe_marker_respects_length_cap() -> None:
|
||||
used: set[str] = set()
|
||||
long_name = "y" * 31
|
||||
assert sanitize_sheet_name(long_name, used) == long_name
|
||||
assert sanitize_sheet_name(long_name, used) == "y" * 29 + "~2"
|
||||
|
||||
|
||||
def test_sheet_name_blank_falls_back() -> None:
|
||||
assert sanitize_sheet_name(" ", set()) == "Sheet"
|
||||
|
||||
|
||||
def test_sheet_name_reserved_history_is_escaped() -> None:
|
||||
assert sanitize_sheet_name("History", set()) == "History_"
|
||||
|
||||
|
||||
def test_sheet_name_strips_surrounding_apostrophes() -> None:
|
||||
assert sanitize_sheet_name("'quoted'", set()) == "quoted"
|
||||
|
||||
|
||||
# --- _sanitize_cell ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected",
|
||||
[
|
||||
(None, ""),
|
||||
("=SUM(A1)", "'=SUM(A1)"),
|
||||
("+1", "'+1"),
|
||||
("-1", "'-1"),
|
||||
("@handle", "'@handle"),
|
||||
("normal", "normal"),
|
||||
(True, True),
|
||||
(5, 5),
|
||||
(1.5, 1.5),
|
||||
(Decimal("2.5"), 2.5),
|
||||
(datetime(2020, 1, 2, 3, 4, 5), "2020-01-02T03:04:05"),
|
||||
(date(2020, 1, 2), "2020-01-02"),
|
||||
],
|
||||
)
|
||||
def test_sanitize_cell(value: object, expected: object) -> None:
|
||||
assert _sanitize_cell(value) == expected
|
||||
|
||||
|
||||
def test_sanitize_cell_large_int_becomes_string() -> None:
|
||||
assert _sanitize_cell(10**16) == str(10**16)
|
||||
|
||||
|
||||
def test_sanitize_cell_non_finite_floats_blanked() -> None:
|
||||
assert _sanitize_cell(float("nan")) == ""
|
||||
assert _sanitize_cell(float("inf")) == ""
|
||||
|
||||
|
||||
def test_sanitize_cell_non_finite_decimals_blanked() -> None:
|
||||
# float(Decimal("NaN")) is nan and float(Decimal("Infinity")) is inf, both
|
||||
# of which xlsxwriter rejects; they must be blanked rather than crash.
|
||||
assert _sanitize_cell(Decimal("NaN")) == ""
|
||||
assert _sanitize_cell(Decimal("Infinity")) == ""
|
||||
assert _sanitize_cell(Decimal("-Infinity")) == ""
|
||||
|
||||
|
||||
def test_sanitize_cell_out_of_range_decimal_is_blanked() -> None:
|
||||
# A Decimal too large for a float becomes inf (or, in edge cases, raises
|
||||
# OverflowError); either way it must be neutralized rather than crash
|
||||
# xlsxwriter or emit a bogus value.
|
||||
assert _sanitize_cell(Decimal("1E10000")) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected",
|
||||
[
|
||||
(" =SUM(A1)", "' =SUM(A1)"),
|
||||
("\t=SUM(A1)", "'\t=SUM(A1)"),
|
||||
(" +1", "' +1"),
|
||||
("\t@handle", "'\t@handle"),
|
||||
],
|
||||
)
|
||||
def test_sanitize_cell_quotes_formula_behind_whitespace(
|
||||
value: str, expected: str
|
||||
) -> None:
|
||||
# Spreadsheet apps evaluate formulas even when preceded by spaces/tabs, so
|
||||
# the formula guard must look past leading whitespace.
|
||||
assert _sanitize_cell(value) == expected
|
||||
|
||||
|
||||
# --- StreamingXlsxWriter (round-trip via openpyxl) ---
|
||||
|
||||
|
||||
def _read_workbook(path: str) -> dict[str, list[list[object]]]:
|
||||
openpyxl = pytest.importorskip("openpyxl")
|
||||
workbook = openpyxl.load_workbook(path, read_only=True)
|
||||
sheets = {
|
||||
ws.title: [list(row) for row in ws.iter_rows(values_only=True)]
|
||||
for ws in workbook.worksheets
|
||||
}
|
||||
workbook.close()
|
||||
return sheets
|
||||
|
||||
|
||||
def test_writer_writes_one_sheet_per_chart(tmp_path: Path) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
assert writer.add_sheet("10 - First", ["a", "b"], [[1, 2], [3, 4]]) == 2
|
||||
assert writer.add_sheet("20 - Second", ["c"], [["x"]]) == 1
|
||||
writer.close()
|
||||
|
||||
sheets = _read_workbook(path)
|
||||
assert list(sheets.keys()) == ["10 - First", "20 - Second"]
|
||||
assert sheets["10 - First"] == [["a", "b"], [1, 2], [3, 4]]
|
||||
assert sheets["20 - Second"] == [["c"], ["x"]]
|
||||
|
||||
|
||||
def test_writer_quotes_formula_cells(tmp_path: Path) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
writer.add_sheet("data", ["col"], [["=cmd()"]])
|
||||
writer.close()
|
||||
|
||||
sheets = _read_workbook(path)
|
||||
assert sheets["data"][1][0] == "'=cmd()"
|
||||
|
||||
|
||||
def test_writer_caps_rows_per_sheet(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(excel_streaming, "MAX_DATA_ROWS_PER_SHEET", 3)
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
written = writer.add_sheet("data", ["col"], [[i] for i in range(5)])
|
||||
writer.close()
|
||||
|
||||
# One row is reserved for the truncation notice, so only 2 data rows fit.
|
||||
assert written == 2
|
||||
sheets = _read_workbook(path)
|
||||
# header + 2 data rows + 1 truncation notice
|
||||
assert len(sheets["data"]) == 4
|
||||
assert sheets["data"][-1][0] == "[Truncated: only first 2 rows exported]"
|
||||
|
||||
|
||||
def test_writer_no_truncation_notice_when_data_fits(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(excel_streaming, "MAX_DATA_ROWS_PER_SHEET", 3)
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
# Exactly fills the reserved capacity (MAX - 1) with no leftover rows.
|
||||
written = writer.add_sheet("data", ["col"], [[i] for i in range(2)])
|
||||
writer.close()
|
||||
|
||||
assert written == 2
|
||||
sheets = _read_workbook(path)
|
||||
# header + 2 data rows, no notice
|
||||
assert sheets["data"] == [["col"], [0], [1]]
|
||||
|
||||
|
||||
def test_writer_empty_workbook_is_valid(tmp_path: Path) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
writer.close()
|
||||
|
||||
sheets = _read_workbook(path)
|
||||
assert list(sheets.keys()) == ["Export"]
|
||||
|
||||
|
||||
def test_writer_summary_sheet(tmp_path: Path) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
writer.add_summary_sheet("Export Summary", ["Skipped charts:", "10 - Broken"])
|
||||
writer.close()
|
||||
|
||||
sheets = _read_workbook(path)
|
||||
assert sheets["Export Summary"] == [["Skipped charts:"], ["10 - Broken"]]
|
||||
|
||||
|
||||
# --- add_image_sheet ---
|
||||
|
||||
# A minimal valid 1x1 transparent PNG.
|
||||
_PNG_1x1 = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
|
||||
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc\x00\x01\x00\x00\x05\x00"
|
||||
b"\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
|
||||
|
||||
def _read_media(path: str) -> list[str]:
|
||||
"""Return the embedded media entries of an xlsx (which is a zip archive)."""
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
return [n for n in archive.namelist() if n.startswith("xl/media/")]
|
||||
|
||||
|
||||
def test_add_image_sheet_embeds_image_and_counts(tmp_path: Path) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
writer.add_image_sheet("10 - Chart", _PNG_1x1)
|
||||
assert writer.sheet_count == 1
|
||||
writer.close()
|
||||
|
||||
# The sheet exists (with a sanitized/unique name) and the PNG is embedded.
|
||||
sheets = _read_workbook(path)
|
||||
assert list(sheets.keys()) == ["10 - Chart"]
|
||||
assert _read_media(path) == ["xl/media/image1.png"]
|
||||
|
||||
|
||||
def test_add_image_sheet_dedupes_and_composes_with_data_sheets(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
path = str(tmp_path / "out.xlsx")
|
||||
writer = StreamingXlsxWriter(path)
|
||||
writer.add_sheet("10 - Chart", ["a"], [[1]])
|
||||
writer.add_image_sheet("10 - Chart", _PNG_1x1)
|
||||
writer.close()
|
||||
|
||||
assert writer.sheet_count == 2
|
||||
sheets = _read_workbook(path)
|
||||
# The image sheet name is de-duplicated against the existing data sheet.
|
||||
assert list(sheets.keys()) == ["10 - Chart", "10 - Chart~2"]
|
||||
assert _read_media(path) == ["xl/media/image1.png"]
|
||||
95
tests/unit_tests/utils/s3_tests.py
Normal file
95
tests/unit_tests/utils/s3_tests.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils import s3
|
||||
|
||||
|
||||
@patch("boto3.client")
|
||||
@patch("superset.utils.s3.current_app")
|
||||
def test_upload_file_to_s3(mock_app: MagicMock, mock_client_fn: MagicMock) -> None:
|
||||
mock_app.config = {"EXCEL_EXPORT_S3_CLIENT_KWARGS": {}}
|
||||
client = mock_client_fn.return_value
|
||||
|
||||
s3.upload_file_to_s3("exports/out.xlsx", "my-bucket", "exports/1/abc.xlsx")
|
||||
|
||||
mock_client_fn.assert_called_once_with("s3")
|
||||
client.upload_file.assert_called_once_with(
|
||||
"exports/out.xlsx", "my-bucket", "exports/1/abc.xlsx"
|
||||
)
|
||||
|
||||
|
||||
@patch("boto3.client")
|
||||
@patch("superset.utils.s3.current_app")
|
||||
def test_client_kwargs_passthrough(
|
||||
mock_app: MagicMock, mock_client_fn: MagicMock
|
||||
) -> None:
|
||||
mock_app.config = {
|
||||
"EXCEL_EXPORT_S3_CLIENT_KWARGS": {
|
||||
"endpoint_url": "http://minio:9000",
|
||||
"region_name": "us-east-1",
|
||||
}
|
||||
}
|
||||
|
||||
s3.upload_file_to_s3("exports/out.xlsx", "my-bucket", "k")
|
||||
|
||||
mock_client_fn.assert_called_once_with(
|
||||
"s3", endpoint_url="http://minio:9000", region_name="us-east-1"
|
||||
)
|
||||
|
||||
|
||||
def test_importing_module_does_not_require_boto3() -> None:
|
||||
# Regression: importing this module (which app startup does via the dashboard
|
||||
# API) must not require boto3, since it is only an optional install.
|
||||
import importlib
|
||||
|
||||
with patch.dict(sys.modules, {"boto3": None}):
|
||||
importlib.reload(s3)
|
||||
# Reload again with boto3 available so later tests see the normal module.
|
||||
importlib.reload(s3)
|
||||
|
||||
|
||||
def test_get_s3_client_missing_boto3_raises_actionable_error() -> None:
|
||||
# Simulate a production install without boto3: the lazy import fails and we
|
||||
# surface an install hint instead of a bare ModuleNotFoundError.
|
||||
with (
|
||||
patch.dict(sys.modules, {"boto3": None}),
|
||||
pytest.raises(ImportError, match="excel-export"),
|
||||
):
|
||||
s3._get_s3_client()
|
||||
|
||||
|
||||
@patch("boto3.client")
|
||||
@patch("superset.utils.s3.current_app")
|
||||
def test_generate_presigned_url(mock_app: MagicMock, mock_client_fn: MagicMock) -> None:
|
||||
mock_app.config = {"EXCEL_EXPORT_S3_CLIENT_KWARGS": {}}
|
||||
client = mock_client_fn.return_value
|
||||
client.generate_presigned_url.return_value = "https://signed.example/abc"
|
||||
|
||||
url = s3.generate_presigned_url("my-bucket", "exports/1/abc.xlsx", 86400)
|
||||
|
||||
assert url == "https://signed.example/abc"
|
||||
client.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={"Bucket": "my-bucket", "Key": "exports/1/abc.xlsx"},
|
||||
ExpiresIn=86400,
|
||||
)
|
||||
Reference in New Issue
Block a user