mirror of
https://github.com/apache/superset.git
synced 2026-07-12 09:45:43 +00:00
Compare commits
4 Commits
fix/dropdo
...
fix-transl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
42379a8253 | ||
|
|
d62a1771e9 | ||
|
|
ad1c976305 | ||
|
|
efeb981626 |
@@ -51,18 +51,8 @@ test('renders children with custom horizontal spacing', () => {
|
||||
expect(screen.getByTestId('container')).toHaveStyle('gap: 20px');
|
||||
});
|
||||
|
||||
test('renders dropdown button when items exist even when not overflowing', () => {
|
||||
test('does not render a dropdown button when not overflowing', () => {
|
||||
render(<DropdownContainer items={generateItems(3)} />);
|
||||
// Button should always be visible when items exist to prevent layout shifts
|
||||
expect(screen.getByText('More')).toBeInTheDocument();
|
||||
// Badge should show 0 when nothing is overflowing
|
||||
expect(screen.getByText('0')).toBeInTheDocument();
|
||||
// Button is disabled when there is nothing to open, so it can't reveal an empty popover
|
||||
expect(screen.getByTestId('dropdown-container-btn')).toBeDisabled();
|
||||
});
|
||||
|
||||
test('does not render a dropdown button when no items', () => {
|
||||
render(<DropdownContainer items={[]} />);
|
||||
expect(screen.queryByText('More')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ import { t } from '@apache-superset/core/translation';
|
||||
import { usePrevious } from '@superset-ui/core';
|
||||
import { css, useTheme } from '@apache-superset/core/theme';
|
||||
import { useResizeDetector } from 'react-resize-detector';
|
||||
import { Badge, Icons, Button, Popover } from '..';
|
||||
import { Badge, Icons, Button, Tooltip, Popover } from '..';
|
||||
import { DropdownContainerProps, DropdownItem, DropdownRef } from './types';
|
||||
|
||||
const MAX_HEIGHT = 500;
|
||||
@@ -72,6 +72,15 @@ export const DropdownContainer = forwardRef(
|
||||
|
||||
const [showOverflow, setShowOverflow] = useState(false);
|
||||
|
||||
// When the item set changes, the overflow index is briefly reset while the
|
||||
// new widths are measured (see the layout effect below). During that window
|
||||
// the dropdown content momentarily becomes empty, which would hide and then
|
||||
// re-show the trigger, causing a flicker. We track whether a recalculation
|
||||
// is pending so the trigger can stay mounted across the transient (when it
|
||||
// was showing content just before) without lingering in the steady state
|
||||
// when nothing actually overflows.
|
||||
const [recalculating, setRecalculating] = useState(false);
|
||||
|
||||
// callback to update item widths so that the useLayoutEffect runs whenever
|
||||
// width of any of the child changes
|
||||
const recalculateItemWidths = useCallback(() => {
|
||||
@@ -171,6 +180,7 @@ export const DropdownContainer = forwardRef(
|
||||
);
|
||||
} else {
|
||||
setOverflowingIndex(-1);
|
||||
setRecalculating(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -211,6 +221,7 @@ export const DropdownContainer = forwardRef(
|
||||
}
|
||||
|
||||
setOverflowingIndex(newOverflowingIndex);
|
||||
setRecalculating(false);
|
||||
}
|
||||
}, [
|
||||
current,
|
||||
@@ -234,13 +245,6 @@ export const DropdownContainer = forwardRef(
|
||||
const overflowingCount =
|
||||
overflowingIndex !== -1 ? items.length - overflowingIndex : 0;
|
||||
|
||||
// Always show button when items exist to prevent layout shifts
|
||||
// and ensure consistent UI even when no items are overflowing.
|
||||
// When items exist but nothing overflows, the button is rendered
|
||||
// disabled (not hidden) with a 0 badge so the container width stays
|
||||
// constant across overflow state changes.
|
||||
const shouldShowButton = items.length > 0 || !!dropdownContent;
|
||||
|
||||
const popoverContent = useMemo(
|
||||
() =>
|
||||
dropdownContent || overflowingCount ? (
|
||||
@@ -268,6 +272,15 @@ export const DropdownContainer = forwardRef(
|
||||
],
|
||||
);
|
||||
|
||||
// The trigger had content in the previous render if popoverContent was
|
||||
// truthy then. During the brief mid-recalculation render where
|
||||
// popoverContent flips to null, this still reflects the prior (non-empty)
|
||||
// value, letting us keep the trigger mounted across the transient.
|
||||
const hadPopoverContent = usePrevious(!!popoverContent, false);
|
||||
|
||||
const showDropdownButton =
|
||||
!!popoverContent || (recalculating && hadPopoverContent);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
if (popoverVisible) {
|
||||
// Measures scroll height after rendering the elements
|
||||
@@ -300,44 +313,6 @@ export const DropdownContainer = forwardRef(
|
||||
};
|
||||
}, [popoverVisible]);
|
||||
|
||||
const triggerButton = (
|
||||
<Button
|
||||
buttonStyle="secondary"
|
||||
data-test="dropdown-container-btn"
|
||||
icon={dropdownTriggerIcon}
|
||||
disabled={!popoverContent}
|
||||
tooltip={dropdownTriggerTooltip}
|
||||
css={css`
|
||||
padding-left: ${theme.paddingXS}px;
|
||||
padding-right: ${theme.paddingXXS}px;
|
||||
gap: ${theme.sizeXXS}px;
|
||||
`}
|
||||
>
|
||||
{dropdownTriggerText}
|
||||
<Badge
|
||||
count={dropdownTriggerCount ?? overflowingCount}
|
||||
color={
|
||||
(dropdownTriggerCount ?? overflowingCount) > 0
|
||||
? theme.colorPrimary
|
||||
: theme.colorTextSecondary
|
||||
}
|
||||
showZero
|
||||
css={css`
|
||||
margin-left: ${theme.sizeUnit * 2}px;
|
||||
`}
|
||||
/>
|
||||
<Icons.DownOutlined
|
||||
iconSize="m"
|
||||
iconColor={theme.colorIcon}
|
||||
css={css`
|
||||
.anticon {
|
||||
display: flex;
|
||||
}
|
||||
`}
|
||||
/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
@@ -359,7 +334,7 @@ export const DropdownContainer = forwardRef(
|
||||
>
|
||||
{notOverflowedItems.map(item => item.element)}
|
||||
</div>
|
||||
{shouldShowButton && (
|
||||
{showDropdownButton && (
|
||||
<>
|
||||
<Global
|
||||
styles={css`
|
||||
@@ -384,27 +359,62 @@ export const DropdownContainer = forwardRef(
|
||||
`}
|
||||
/>
|
||||
|
||||
{popoverContent ? (
|
||||
<Popover
|
||||
styles={{
|
||||
body: {
|
||||
maxHeight: `${MAX_HEIGHT}px`,
|
||||
overflow: showOverflow ? 'auto' : 'visible',
|
||||
},
|
||||
}}
|
||||
content={popoverContent}
|
||||
trigger="click"
|
||||
open={popoverVisible}
|
||||
onOpenChange={visible => setPopoverVisible(visible)}
|
||||
placement="bottom"
|
||||
forceRender={forceRender}
|
||||
fresh // This prop prevents caching and stale data for filter scoping.
|
||||
>
|
||||
{triggerButton}
|
||||
</Popover>
|
||||
) : (
|
||||
triggerButton
|
||||
)}
|
||||
<Popover
|
||||
styles={{
|
||||
body: {
|
||||
maxHeight: `${MAX_HEIGHT}px`,
|
||||
overflow: showOverflow ? 'auto' : 'visible',
|
||||
},
|
||||
}}
|
||||
content={popoverContent}
|
||||
trigger="click"
|
||||
open={popoverVisible && !!popoverContent}
|
||||
onOpenChange={visible => {
|
||||
// While a recalculation keeps the trigger mounted but there is
|
||||
// no content yet, ignore open attempts so it stays visible
|
||||
// without opening an empty popover.
|
||||
if (popoverContent) setPopoverVisible(visible);
|
||||
}}
|
||||
placement="bottom"
|
||||
forceRender={forceRender}
|
||||
fresh // This prop prevents caching and stale data for filter scoping.
|
||||
>
|
||||
<Tooltip title={dropdownTriggerTooltip}>
|
||||
<Button
|
||||
buttonStyle="secondary"
|
||||
data-test="dropdown-container-btn"
|
||||
icon={dropdownTriggerIcon}
|
||||
css={css`
|
||||
padding-left: ${theme.paddingXS}px;
|
||||
padding-right: ${theme.paddingXXS}px;
|
||||
gap: ${theme.sizeXXS}px;
|
||||
`}
|
||||
>
|
||||
{dropdownTriggerText}
|
||||
<Badge
|
||||
count={dropdownTriggerCount ?? overflowingCount}
|
||||
color={
|
||||
(dropdownTriggerCount ?? overflowingCount) > 0
|
||||
? theme.colorPrimary
|
||||
: theme.colorTextSecondary
|
||||
}
|
||||
showZero
|
||||
css={css`
|
||||
margin-left: ${theme.sizeUnit * 2}px;
|
||||
`}
|
||||
/>
|
||||
<Icons.DownOutlined
|
||||
iconSize="m"
|
||||
iconColor={theme.colorIcon}
|
||||
css={css`
|
||||
.anticon {
|
||||
display: flex;
|
||||
}
|
||||
`}
|
||||
/>
|
||||
</Button>
|
||||
</Tooltip>
|
||||
</Popover>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
} from '@apache-superset/core/theme';
|
||||
import Switchboard from '@superset-ui/switchboard';
|
||||
import getBootstrapData, { applicationRoot } from 'src/utils/getBootstrapData';
|
||||
import initPreamble from 'src/preamble';
|
||||
import setupClient from 'src/setup/setupClient';
|
||||
import setupPlugins from 'src/setup/setupPlugins';
|
||||
import { useUiConfig } from 'src/components/UiConfigContext';
|
||||
@@ -50,8 +51,19 @@ import { embeddedApi } from './api';
|
||||
import { getDataMaskChangeTrigger } from './utils';
|
||||
import { validateMessageEvent } from './originValidation';
|
||||
|
||||
setupPlugins();
|
||||
setupCodeOverrides({ embedded: true });
|
||||
// Defer plugin setup until after the language pack loads to prevent t() calls in
|
||||
// plugin control panel configs from being cached in English before translations are ready.
|
||||
const pluginsReady = initPreamble()
|
||||
.catch(err => {
|
||||
logging.warn(
|
||||
'Preamble initialization failed, loading plugins without translations.',
|
||||
err,
|
||||
);
|
||||
})
|
||||
.then(() => {
|
||||
setupPlugins();
|
||||
setupCodeOverrides({ embedded: true });
|
||||
});
|
||||
|
||||
const debugMode = process.env.WEBPACK_MODE === 'development';
|
||||
const bootstrapData = getBootstrapData();
|
||||
@@ -172,32 +184,34 @@ function start() {
|
||||
method: 'GET',
|
||||
endpoint: '/api/v1/me/roles/',
|
||||
});
|
||||
return getMeWithRole().then(
|
||||
({ result }) => {
|
||||
// fill in some missing bootstrap data
|
||||
// (because at pageload, we don't have any auth yet)
|
||||
// this allows the frontend's permissions checks to work.
|
||||
bootstrapData.user = result;
|
||||
store.dispatch({
|
||||
type: USER_LOADED,
|
||||
user: result,
|
||||
});
|
||||
if (!root) {
|
||||
root = createRoot(appMountPoint);
|
||||
}
|
||||
root.render(<EmbeddedApp />);
|
||||
},
|
||||
err => {
|
||||
// something is most likely wrong with the guest token; reset the guard
|
||||
// so a rehandshake with a valid token can retry.
|
||||
logging.error(err);
|
||||
showFailureMessage(
|
||||
t(
|
||||
'Something went wrong with embedded authentication. Check the dev console for details.',
|
||||
),
|
||||
);
|
||||
started = false;
|
||||
},
|
||||
return pluginsReady.then(() =>
|
||||
getMeWithRole().then(
|
||||
({ result }) => {
|
||||
// fill in some missing bootstrap data
|
||||
// (because at pageload, we don't have any auth yet)
|
||||
// this allows the frontend's permissions checks to work.
|
||||
bootstrapData.user = result;
|
||||
store.dispatch({
|
||||
type: USER_LOADED,
|
||||
user: result,
|
||||
});
|
||||
if (!root) {
|
||||
root = createRoot(appMountPoint);
|
||||
}
|
||||
root.render(<EmbeddedApp />);
|
||||
},
|
||||
err => {
|
||||
// something is most likely wrong with the guest token; reset the guard
|
||||
// so a rehandshake with a valid token can retry.
|
||||
logging.error(err);
|
||||
showFailureMessage(
|
||||
t(
|
||||
'Something went wrong with embedded authentication. Check the dev console for details.',
|
||||
),
|
||||
);
|
||||
started = false;
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ import { ThemeProvider } from '@apache-superset/core/theme';
|
||||
import { theme } from '@apache-superset/core/theme';
|
||||
import Menu from 'src/features/home/Menu';
|
||||
import getBootstrapData from 'src/utils/getBootstrapData';
|
||||
import initPreamble from 'src/preamble';
|
||||
import { setupStore } from './store';
|
||||
import querystring from 'query-string';
|
||||
|
||||
@@ -67,5 +68,9 @@ const app = (
|
||||
|
||||
const menuMountPoint = document.getElementById('app-menu');
|
||||
if (menuMountPoint) {
|
||||
createRoot(menuMountPoint).render(app);
|
||||
initPreamble()
|
||||
.catch(() => {}) // preamble logs failures internally; always render the menu
|
||||
.then(() => {
|
||||
createRoot(menuMountPoint).render(app);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -389,9 +389,7 @@ def apply_client_processing( # noqa: C901
|
||||
query["data"] = processed_df.to_dict()
|
||||
elif query["result_format"] == ChartDataResultFormat.CSV:
|
||||
buf = StringIO()
|
||||
# Apply CSV_EXPORT config for consistent CSV formatting
|
||||
csv_export_config = current_app.config["CSV_EXPORT"]
|
||||
processed_df.to_csv(buf, index=show_default_index, **csv_export_config)
|
||||
processed_df.to_csv(buf, index=show_default_index)
|
||||
buf.seek(0)
|
||||
query["data"] = buf.getvalue()
|
||||
|
||||
|
||||
@@ -24,8 +24,6 @@ import logging
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from decimal import Decimal
|
||||
from numbers import Real
|
||||
from typing import Any, Callable, Generator
|
||||
|
||||
from flask import current_app as app, g, has_app_context
|
||||
@@ -109,55 +107,16 @@ class BaseStreamingCSVExportCommand(BaseCommand):
|
||||
buffer.truncate()
|
||||
return header_data, total_bytes
|
||||
|
||||
def _format_row_values(
|
||||
self, row: tuple[Any, ...], decimal_separator: str | None
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Format row values, applying custom decimal separator if specified.
|
||||
|
||||
Args:
|
||||
row: Database row as a tuple
|
||||
decimal_separator: Custom decimal separator (e.g., ",") or None
|
||||
|
||||
Returns:
|
||||
List of formatted values
|
||||
"""
|
||||
if not decimal_separator or decimal_separator == ".":
|
||||
return list(row)
|
||||
|
||||
formatted: list[Any] = []
|
||||
for value in row:
|
||||
# Apply the custom decimal separator to any real numeric value
|
||||
# (float, decimal.Decimal, numpy numeric types, ...). Booleans are
|
||||
# technically a numeric type in Python but should never be rewritten
|
||||
# as numbers in CSV output.
|
||||
if isinstance(value, bool):
|
||||
formatted.append(value)
|
||||
elif isinstance(value, (float, Decimal, Real)):
|
||||
# Format numeric values with custom decimal separator
|
||||
formatted.append(str(value).replace(".", decimal_separator))
|
||||
else:
|
||||
formatted.append(value)
|
||||
return formatted
|
||||
|
||||
def _process_rows(
|
||||
self,
|
||||
result_proxy: Any,
|
||||
csv_writer: Any,
|
||||
buffer: io.StringIO,
|
||||
limit: int | None,
|
||||
decimal_separator: str | None = None,
|
||||
) -> Generator[tuple[str, int, int], None, None]:
|
||||
"""
|
||||
Process database rows and yield CSV data chunks.
|
||||
|
||||
Args:
|
||||
result_proxy: SQLAlchemy result proxy
|
||||
csv_writer: CSV writer instance
|
||||
buffer: StringIO buffer for CSV data
|
||||
limit: Maximum number of rows to process, or None for unlimited
|
||||
decimal_separator: Custom decimal separator (e.g., ",") or None
|
||||
|
||||
Yields tuples of (data_chunk, row_count, byte_count).
|
||||
"""
|
||||
row_count = 0
|
||||
@@ -169,9 +128,7 @@ class BaseStreamingCSVExportCommand(BaseCommand):
|
||||
if limit is not None and row_count >= limit:
|
||||
break
|
||||
|
||||
# Format values with custom decimal separator if needed
|
||||
formatted_row = self._format_row_values(row, decimal_separator)
|
||||
csv_writer.writerow(formatted_row)
|
||||
csv_writer.writerow(row)
|
||||
row_count += 1
|
||||
|
||||
# Check buffer size and flush if needed
|
||||
@@ -204,21 +161,6 @@ class BaseStreamingCSVExportCommand(BaseCommand):
|
||||
start_time = time.time()
|
||||
total_bytes = 0
|
||||
|
||||
# Get CSV export configuration. CSV_EXPORT has an explicit default in
|
||||
# config.py, so index directly rather than using .get() with a hardcoded
|
||||
# fallback that would silently mask a misconfiguration removing the key.
|
||||
#
|
||||
# The streaming path only honors the `sep` and `decimal` keys from
|
||||
# CSV_EXPORT. Unlike the non-streaming path in
|
||||
# superset.charts.client_processing (which builds the whole file with a
|
||||
# single DataFrame.to_csv(**CSV_EXPORT) call), this path writes rows
|
||||
# incrementally via csv.writer, so the remaining pandas to_csv kwargs
|
||||
# (e.g. quotechar, lineterminator, encoding) do not map onto it and are
|
||||
# intentionally not applied here.
|
||||
csv_export_config = app.config["CSV_EXPORT"]
|
||||
delimiter = csv_export_config.get("sep", ",")
|
||||
decimal_separator = csv_export_config.get("decimal", ".")
|
||||
|
||||
with db.session() as session:
|
||||
# Merge database to prevent DetachedInstanceError
|
||||
merged_database = session.merge(database)
|
||||
@@ -234,11 +176,8 @@ class BaseStreamingCSVExportCommand(BaseCommand):
|
||||
columns = list(result_proxy.keys())
|
||||
|
||||
# Use StringIO with csv.writer for proper escaping
|
||||
# Apply delimiter from CSV_EXPORT config
|
||||
buffer = io.StringIO()
|
||||
csv_writer = csv.writer(
|
||||
buffer, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL
|
||||
)
|
||||
csv_writer = csv.writer(buffer, quoting=csv.QUOTE_MINIMAL)
|
||||
|
||||
# Write CSV header
|
||||
header_data, header_bytes = self._write_csv_header(
|
||||
@@ -250,7 +189,7 @@ class BaseStreamingCSVExportCommand(BaseCommand):
|
||||
# Process rows and yield chunks
|
||||
row_count = 0
|
||||
for data_chunk, rows_processed, chunk_bytes in self._process_rows(
|
||||
result_proxy, csv_writer, buffer, limit, decimal_separator
|
||||
result_proxy, csv_writer, buffer, limit
|
||||
):
|
||||
total_bytes += chunk_bytes
|
||||
row_count = rows_processed
|
||||
|
||||
@@ -1489,11 +1489,6 @@ SQLLAB_QUERY_COST_ESTIMATE_TIMEOUT = int(timedelta(seconds=10).total_seconds())
|
||||
# 0 means no timeout.
|
||||
SQLLAB_QUERY_RESULT_TIMEOUT = 0
|
||||
|
||||
# Connect/read timeout (in seconds) for the synchronous network call made when
|
||||
# detecting ODPS (MaxCompute) partition info during table preview. Prevents an
|
||||
# unreachable or slow ODPS endpoint from blocking the web worker indefinitely.
|
||||
ODPS_PARTITION_DETECT_TIMEOUT = int(timedelta(seconds=30).total_seconds())
|
||||
|
||||
# The cost returned by the databases is a relative value; in order to map the cost to
|
||||
# a tangible value you need to define a custom formatter that takes into consideration
|
||||
# your specific infrastructure. For example, you could analyze queries a posteriori by
|
||||
|
||||
@@ -17,22 +17,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
from flask import current_app as app
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
try:
|
||||
from odps import ODPS, options as odps_options
|
||||
from odps.errors import BaseODPSError
|
||||
except ImportError:
|
||||
ODPS = None
|
||||
odps_options = None
|
||||
BaseODPSError = None
|
||||
|
||||
from superset import is_feature_enabled
|
||||
from superset.commands.database.ssh_tunnel.exceptions import SSHTunnelingNotEnabledError
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
@@ -261,69 +249,6 @@ class DatabaseDAO(BaseDAO[Database]):
|
||||
.all()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_odps_partitioned_table(
|
||||
cls, database: Database, table_name: str
|
||||
) -> tuple[bool, list[str]]:
|
||||
"""
|
||||
This function is used to determine and retrieve
|
||||
partition information of the ODPS table.
|
||||
The return values are whether the partition
|
||||
table is partitioned and the names of all partition fields.
|
||||
"""
|
||||
if not database:
|
||||
raise ValueError("Database not found")
|
||||
if database.backend != "odps":
|
||||
return False, []
|
||||
if ODPS is None:
|
||||
logger.warning("pyodps is not installed, cannot check ODPS partition info")
|
||||
return False, []
|
||||
uri = database.sqlalchemy_uri
|
||||
access_key = database.password
|
||||
pattern = re.compile(
|
||||
r"odps://(?P<username>[^:]+):(?P<password>[^@]+)@(?P<project>[^/]+)/(?:\?"
|
||||
r"endpoint=(?P<endpoint>[^&]+))"
|
||||
)
|
||||
if not uri or not isinstance(uri, str):
|
||||
logger.warning(
|
||||
"Invalid or missing sqlalchemy URI, please provide a correct URI"
|
||||
)
|
||||
return False, []
|
||||
if match := pattern.match(unquote(uri)):
|
||||
access_id = match.group("username")
|
||||
project = match.group("project")
|
||||
endpoint = match.group("endpoint")
|
||||
# `get_table` is a synchronous network call. Bound it with a
|
||||
# configurable connect/read timeout so an unreachable or slow ODPS
|
||||
# endpoint can't block the worker indefinitely.
|
||||
timeout = app.config["ODPS_PARTITION_DETECT_TIMEOUT"]
|
||||
if odps_options is not None:
|
||||
odps_options.connect_timeout = timeout
|
||||
odps_options.read_timeout = timeout
|
||||
try:
|
||||
odps_client = ODPS(access_id, access_key, project, endpoint=endpoint)
|
||||
table = odps_client.get_table(table_name)
|
||||
if table.exist_partition:
|
||||
partition_spec = table.table_schema.partitions
|
||||
partition_fields = [partition.name for partition in partition_spec]
|
||||
return True, partition_fields
|
||||
return False, []
|
||||
except (BaseODPSError, requests.exceptions.RequestException) as ex:
|
||||
# Network/auth/lookup failures against ODPS shouldn't break table
|
||||
# preview; fall back to the non-partitioned path.
|
||||
logger.warning(
|
||||
"Error fetching ODPS partition info for table %r: %s",
|
||||
table_name,
|
||||
ex,
|
||||
)
|
||||
return False, []
|
||||
logger.warning(
|
||||
"ODPS sqlalchemy_uri did not match the expected pattern; "
|
||||
"unable to determine partition info for table %r",
|
||||
table_name,
|
||||
)
|
||||
return False, []
|
||||
|
||||
|
||||
class DatabaseUserOAuth2TokensDAO(BaseDAO[DatabaseUserOAuth2Tokens]):
|
||||
"""
|
||||
|
||||
@@ -123,7 +123,7 @@ from superset.exceptions import (
|
||||
)
|
||||
from superset.extensions import security_manager
|
||||
from superset.models.core import Database
|
||||
from superset.sql.parse import Partition, Table
|
||||
from superset.sql.parse import Table
|
||||
from superset.superset_typing import FlaskResponse
|
||||
from superset.utils import json
|
||||
from superset.utils.core import (
|
||||
@@ -1079,25 +1079,15 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
parameters = QualifiedTableSchema().load(request.args)
|
||||
except ValidationError as ex:
|
||||
raise InvalidPayloadSchemaError(ex) from ex
|
||||
table_name = str(parameters["name"])
|
||||
table = Table(table_name, parameters["schema"], parameters["catalog"])
|
||||
|
||||
table = Table(parameters["name"], parameters["schema"], parameters["catalog"])
|
||||
try:
|
||||
security_manager.raise_for_access(database=database, table=table)
|
||||
except SupersetSecurityException as ex:
|
||||
# instead of raising 403, raise 404 to hide table existence
|
||||
raise TableNotFoundException("No such table") from ex
|
||||
# `is_odps_partitioned_table` returns (False, []) for non-ODPS backends
|
||||
# and handles its own optional-dependency / network / auth failures
|
||||
# internally, so any exception escaping here is an unexpected programming
|
||||
# error that should propagate rather than be silently swallowed.
|
||||
is_partitioned_table, partition_fields = DatabaseDAO.is_odps_partitioned_table(
|
||||
database, table_name
|
||||
)
|
||||
partition = Partition(is_partitioned_table, tuple(partition_fields))
|
||||
# Partition info is engine-agnostic at this layer: the generic dispatch
|
||||
# passes it to the engine spec, which decides whether to use it. Non-ODPS
|
||||
# specs ignore the parameter.
|
||||
payload = database.db_engine_spec.get_table_metadata(database, table, partition)
|
||||
|
||||
payload = database.db_engine_spec.get_table_metadata(database, table)
|
||||
|
||||
return self.response(200, **payload)
|
||||
|
||||
|
||||
@@ -81,14 +81,6 @@ def load_engine_specs() -> list[type[BaseEngineSpec]]:
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.warning("Unable to load Superset DB engine spec: %s", ep.name)
|
||||
continue
|
||||
# Validate that the engine spec is a proper subclass of BaseEngineSpec
|
||||
if not is_engine_spec(engine_spec):
|
||||
logger.warning(
|
||||
"Skipping invalid DB engine spec %s: "
|
||||
"not a valid BaseEngineSpec subclass",
|
||||
ep.name,
|
||||
)
|
||||
continue
|
||||
engine_specs.append(engine_spec)
|
||||
|
||||
return engine_specs
|
||||
|
||||
@@ -72,7 +72,6 @@ from superset.key_value.types import JsonKeyValueCodec, KeyValueResource
|
||||
from superset.sql.parse import (
|
||||
BaseSQLStatement,
|
||||
LimitMethod,
|
||||
Partition,
|
||||
RLSMethod,
|
||||
SQLScript,
|
||||
SQLStatement,
|
||||
@@ -1356,15 +1355,12 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
cls,
|
||||
database: Database,
|
||||
table: Table,
|
||||
partition: Partition | None = None,
|
||||
) -> TableMetadataResponse:
|
||||
"""
|
||||
Returns basic table metadata
|
||||
|
||||
:param database: Database instance
|
||||
:param table: A Table instance
|
||||
:param partition: Optional partition info used by engines that support
|
||||
partitioned tables (e.g. ODPS). Ignored by engines that don't.
|
||||
:return: Basic table metadata
|
||||
"""
|
||||
return get_table_metadata(database, table)
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, cast, TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.engine import Dialect
|
||||
|
||||
from superset.databases.schemas import (
|
||||
TableMetadataColumnsResponse,
|
||||
TableMetadataPrimaryKeyResponse,
|
||||
TableMetadataResponse,
|
||||
)
|
||||
from superset.databases.utils import (
|
||||
get_col_type,
|
||||
get_foreign_keys_metadata,
|
||||
get_indexes_metadata,
|
||||
)
|
||||
from superset.db_engine_specs.base import BaseEngineSpec, BasicParametersMixin
|
||||
from superset.sql.parse import Partition, SQLScript, Table
|
||||
from superset.superset_typing import ResultSetColumnType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.models.core import Database
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OdpsBaseEngineSpec(BaseEngineSpec):
|
||||
@classmethod
|
||||
def get_table_metadata(
|
||||
cls,
|
||||
database: Database,
|
||||
table: Table,
|
||||
partition: Partition | None = None,
|
||||
) -> TableMetadataResponse:
|
||||
"""
|
||||
Returns basic table metadata
|
||||
:param database: Database instance
|
||||
:param table: A Table instance
|
||||
:param partition: A Table partition info
|
||||
:return: Basic table metadata
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class OdpsEngineSpec(BasicParametersMixin, OdpsBaseEngineSpec):
|
||||
engine = "odps"
|
||||
engine_name = "ODPS (MaxCompute)"
|
||||
default_driver = "odps"
|
||||
|
||||
@classmethod
|
||||
def get_table_metadata(
|
||||
cls, database: Database, table: Table, partition: Partition | None = None
|
||||
) -> TableMetadataResponse:
|
||||
"""
|
||||
Get table metadata information, including type, pk, fks.
|
||||
This function raises SQLAlchemyError when a schema is not found.
|
||||
|
||||
:param partition: The table's partition info
|
||||
:param database: The database model
|
||||
:param table: Table instance
|
||||
:return: Dict table metadata ready for API response
|
||||
"""
|
||||
keys: list[Any] = []
|
||||
columns = database.get_columns(table)
|
||||
primary_key = database.get_pk_constraint(table)
|
||||
if primary_key and primary_key.get("constrained_columns"):
|
||||
primary_key["column_names"] = primary_key.pop("constrained_columns")
|
||||
primary_key["type"] = "pk"
|
||||
keys += [primary_key]
|
||||
foreign_keys = get_foreign_keys_metadata(database, table)
|
||||
indexes = get_indexes_metadata(database, table)
|
||||
keys += foreign_keys + indexes
|
||||
payload_columns: list[TableMetadataColumnsResponse] = []
|
||||
table_comment = database.get_table_comment(table)
|
||||
for col in columns:
|
||||
dtype = get_col_type(cast("dict[Any, Any]", col))
|
||||
payload_columns.append(
|
||||
{
|
||||
"name": col["column_name"],
|
||||
"type": dtype.split("(")[0] if "(" in dtype else dtype,
|
||||
"longType": dtype,
|
||||
"keys": [
|
||||
k for k in keys if col["column_name"] in k["column_names"]
|
||||
],
|
||||
"comment": col.get("comment"),
|
||||
}
|
||||
)
|
||||
|
||||
with database.get_sqla_engine(
|
||||
catalog=table.catalog, schema=table.schema
|
||||
) as engine:
|
||||
return {
|
||||
"name": table.table,
|
||||
"columns": payload_columns,
|
||||
"selectStar": cls.select_star(
|
||||
database=database,
|
||||
table=table,
|
||||
dialect=engine.dialect,
|
||||
limit=100,
|
||||
show_cols=False,
|
||||
indent=True,
|
||||
latest_partition=True,
|
||||
cols=columns,
|
||||
partition=partition,
|
||||
),
|
||||
"primaryKey": cast("TableMetadataPrimaryKeyResponse", primary_key),
|
||||
"foreignKeys": foreign_keys,
|
||||
"indexes": keys,
|
||||
"comment": table_comment,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def select_star( # pylint: disable=too-many-arguments
|
||||
cls,
|
||||
database: Database,
|
||||
table: Table,
|
||||
dialect: Dialect,
|
||||
limit: int = 100,
|
||||
show_cols: bool = False,
|
||||
indent: bool = True,
|
||||
latest_partition: bool = True,
|
||||
cols: list[ResultSetColumnType] | None = None,
|
||||
partition: Partition | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a "SELECT * from [schema.]table_name" query with appropriate limit.
|
||||
|
||||
WARNING: expects only unquoted table and schema names.
|
||||
|
||||
:param partition: The table's partition info
|
||||
:param database: Database instance
|
||||
:param table: Table instance
|
||||
:param dialect: SqlAlchemy Dialect instance
|
||||
:param limit: limit to impose on query
|
||||
:param show_cols: Show columns in query; otherwise use "*"
|
||||
:param indent: Add indentation to query
|
||||
:param latest_partition: Only query the latest partition
|
||||
:param cols: Columns to include in query
|
||||
:return: SQL query
|
||||
"""
|
||||
# pylint: disable=redefined-outer-name
|
||||
fields: str | list[Any] = "*"
|
||||
cols = cols or []
|
||||
if (show_cols or latest_partition) and not cols:
|
||||
cols = database.get_columns(table)
|
||||
|
||||
if show_cols:
|
||||
fields = cls._get_fields(cols)
|
||||
full_table_name = cls.quote_table(table, dialect)
|
||||
qry = select(fields).select_from(text(full_table_name))
|
||||
if database.backend == "odps":
|
||||
if (
|
||||
partition is not None
|
||||
and partition.is_partitioned_table
|
||||
and partition.partition_column is not None
|
||||
and len(partition.partition_column) > 0
|
||||
):
|
||||
partition_str = partition.partition_column[0]
|
||||
# `partition_str` is a column name sourced from ODPS schema
|
||||
# metadata, so quote it through the dialect's identifier preparer
|
||||
# to guard against names containing SQL metacharacters.
|
||||
quoted = dialect.identifier_preparer.quote(partition_str)
|
||||
# A match-all `LIKE '%'` predicate is used (rather than a real
|
||||
# filter) purely to force ODPS to scan a partitioned table; the
|
||||
# engine rejects an unpartitioned full-table preview, so this
|
||||
# no-op predicate keeps the preview query valid.
|
||||
partition_str_where = f"CAST({quoted} AS STRING) LIKE '%'"
|
||||
qry = qry.where(text(partition_str_where))
|
||||
if limit:
|
||||
qry = qry.limit(limit)
|
||||
if latest_partition:
|
||||
partition_query = cls.where_latest_partition(
|
||||
database,
|
||||
table,
|
||||
qry,
|
||||
columns=cols,
|
||||
)
|
||||
if partition_query is not None:
|
||||
qry = partition_query
|
||||
sql = database.compile_sqla_query(qry, table.catalog, table.schema)
|
||||
if indent:
|
||||
sql = SQLScript(sql, engine=cls.engine).format()
|
||||
return sql
|
||||
@@ -325,34 +325,6 @@ class Table:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(eq=True, frozen=True)
|
||||
class Partition:
|
||||
"""
|
||||
Partition object, with two attribute keys:
|
||||
is_partitioned_table and partition_column,
|
||||
used to provide partition information
|
||||
Here is an example of an object:
|
||||
Partition(is_partitioned_table=True, partition_column=("month", "day"))
|
||||
"""
|
||||
|
||||
is_partitioned_table: bool
|
||||
partition_column: tuple[str, ...] | None = None
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""
|
||||
Return a string representation of the Partition object.
|
||||
"""
|
||||
partition_column_str = (
|
||||
", ".join(map(str, self.partition_column))
|
||||
if self.partition_column
|
||||
else "None"
|
||||
)
|
||||
return (
|
||||
f"Partition(is_partitioned_table={self.is_partitioned_table}, "
|
||||
f"partition_column=[{partition_column_str}])"
|
||||
)
|
||||
|
||||
|
||||
# To avoid unnecessary parsing/formatting of queries, the statement has the concept of
|
||||
# an "internal representation", which is the AST of the SQL statement. For most of the
|
||||
# engines supported by Superset this is `sqlglot.exp.Expression`, but there is a special
|
||||
|
||||
@@ -27,7 +27,6 @@ from superset.db_engine_specs.base import (
|
||||
builtin_time_grains,
|
||||
)
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
from superset.db_engine_specs.odps import OdpsBaseEngineSpec, OdpsEngineSpec
|
||||
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
from superset.sql.parse import Table
|
||||
@@ -81,11 +80,7 @@ class SupersetTestCases(SupersetTestCase):
|
||||
time_grains = set(builtin_time_grains.keys())
|
||||
# loop over all subclasses of BaseEngineSpec
|
||||
for engine in load_engine_specs():
|
||||
if (
|
||||
engine is not BaseEngineSpec
|
||||
and engine is not OdpsBaseEngineSpec
|
||||
and engine is not OdpsEngineSpec
|
||||
):
|
||||
if engine is not BaseEngineSpec:
|
||||
# make sure time grain functions have been defined
|
||||
assert len(engine.get_time_grain_expressions()) > 0
|
||||
# make sure all defined time grains are supported
|
||||
|
||||
@@ -2788,114 +2788,3 @@ def test_apply_client_processing_csv_format_default_na_behavior():
|
||||
assert (
|
||||
"Alice," in lines[2]
|
||||
) # Second data row should have empty last_name (NA converted to null)
|
||||
|
||||
|
||||
@with_config({"CSV_EXPORT": {"sep": ";", "decimal": ","}})
|
||||
def test_apply_client_processing_csv_format_custom_separator() -> None:
|
||||
"""
|
||||
Test that apply_client_processing respects CSV_EXPORT config
|
||||
for custom separator and decimal character.
|
||||
|
||||
This is a regression test for GitHub issue #32371.
|
||||
"""
|
||||
# CSV data with numeric values
|
||||
csv_data = "name,value\nAlice,1.5\nBob,2.75"
|
||||
|
||||
result = {
|
||||
"queries": [
|
||||
{
|
||||
"result_format": ChartDataResultFormat.CSV,
|
||||
"data": csv_data,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
form_data = {
|
||||
"datasource": "1__table",
|
||||
"viz_type": "table",
|
||||
"slice_id": 1,
|
||||
"url_params": {},
|
||||
"metrics": [],
|
||||
"groupby": [],
|
||||
"columns": ["name", "value"],
|
||||
"extra_form_data": {},
|
||||
"force": False,
|
||||
"result_format": "csv",
|
||||
"result_type": "results",
|
||||
}
|
||||
|
||||
processed_result = apply_client_processing(result, form_data)
|
||||
|
||||
output_data = processed_result["queries"][0]["data"]
|
||||
lines = output_data.strip().split("\n")
|
||||
|
||||
# With sep=";", columns should be separated by semicolon
|
||||
assert lines[0] == "name;value"
|
||||
# With decimal=",", decimal values must use comma as separator.
|
||||
# Asserting the exact formatted value ensures a regression that drops
|
||||
# the `decimal` option (so floats keep a dot) will be caught.
|
||||
assert "Alice;1,5" in lines[1]
|
||||
assert "Bob;2,75" in lines[2]
|
||||
# Guard explicitly against the dot form slipping through.
|
||||
assert "1.5" not in lines[1]
|
||||
assert "2.75" not in lines[2]
|
||||
|
||||
|
||||
@with_config({"CSV_EXPORT": {"sep": ";", "decimal": ","}})
|
||||
def test_apply_client_processing_csv_pivot_table_custom_separator() -> None:
|
||||
"""
|
||||
Test that apply_client_processing respects CSV_EXPORT config
|
||||
for pivot table exports with custom separator and decimal character.
|
||||
|
||||
This is a regression test for GitHub issue #32371 - specifically for
|
||||
pivoted CSV exports which were not respecting the CSV_EXPORT config.
|
||||
"""
|
||||
# CSV data with a numeric metric
|
||||
csv_data = "COUNT(metric)\n1234.56"
|
||||
|
||||
result = {
|
||||
"queries": [
|
||||
{
|
||||
"result_format": ChartDataResultFormat.CSV,
|
||||
"data": csv_data,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
form_data = {
|
||||
"datasource": "1__table",
|
||||
"viz_type": "pivot_table_v2",
|
||||
"slice_id": 1,
|
||||
"url_params": {},
|
||||
"groupbyColumns": [],
|
||||
"groupbyRows": [],
|
||||
"metrics": [
|
||||
{
|
||||
"aggregate": "COUNT",
|
||||
"column": {"column_name": "metric"},
|
||||
"expressionType": "SIMPLE",
|
||||
"label": "COUNT(metric)",
|
||||
}
|
||||
],
|
||||
"metricsLayout": "COLUMNS",
|
||||
"aggregateFunction": "Sum",
|
||||
"extra_form_data": {},
|
||||
"force": False,
|
||||
"result_format": "csv",
|
||||
"result_type": "results",
|
||||
}
|
||||
|
||||
processed_result = apply_client_processing(result, form_data)
|
||||
|
||||
output_data = processed_result["queries"][0]["data"]
|
||||
lines = output_data.strip().split("\n")
|
||||
|
||||
# After pivoting a single metric with no groupby rows/columns, the
|
||||
# CSV for the "COUNT(metric)" column and "Total (Sum)" row should
|
||||
# reflect the CSV_EXPORT config: semicolons as field separators and
|
||||
# commas as the decimal separator.
|
||||
assert lines[0] == ";COUNT(metric)"
|
||||
assert "Total (Sum);1234,56" in lines[1]
|
||||
# Guard explicitly against the dot form slipping through, which is
|
||||
# what the previous (broken) implementation produced.
|
||||
assert "1234.56" not in output_data
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
# under the License.
|
||||
"""Unit tests for SQL Lab Streaming CSV Export Command."""
|
||||
|
||||
from decimal import Decimal
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
@@ -578,196 +577,3 @@ def test_catalog_and_schema_passed_to_engine(mocker, mock_query, mock_result_pro
|
||||
catalog="my_catalog",
|
||||
schema="my_schema",
|
||||
)
|
||||
|
||||
|
||||
def test_csv_export_config_custom_separator(mocker, mock_query) -> None:
|
||||
"""
|
||||
Test that streaming CSV export respects CSV_EXPORT config
|
||||
for custom separator (sep).
|
||||
|
||||
This is a regression test for GitHub issue #32371.
|
||||
"""
|
||||
mock_query.select_sql = "SELECT * FROM test"
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.keys.return_value = ["id", "name"]
|
||||
mock_result.fetchmany.side_effect = [
|
||||
[(1, "Alice"), (2, "Bob")],
|
||||
[],
|
||||
]
|
||||
|
||||
mock_db, mock_session = _setup_sqllab_mocks(mocker, mock_query)
|
||||
|
||||
mock_connection = MagicMock()
|
||||
mock_connection.execution_options.return_value.execute.return_value = mock_result
|
||||
mock_connection.__enter__.return_value = mock_connection
|
||||
mock_connection.__exit__.return_value = None
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value = mock_connection
|
||||
mock_query.database.get_sqla_engine.return_value.__enter__.return_value = (
|
||||
mock_engine
|
||||
)
|
||||
|
||||
# Mock the app config to use semicolon separator
|
||||
mocker.patch.dict(
|
||||
"superset.commands.streaming_export.base.app.config",
|
||||
{"CSV_EXPORT": {"sep": ";", "encoding": "utf-8"}},
|
||||
)
|
||||
|
||||
command = StreamingSqlResultExportCommand("test_client_123")
|
||||
command.validate()
|
||||
|
||||
csv_generator_callable = command.run()
|
||||
generator = csv_generator_callable()
|
||||
csv_data = "".join(generator)
|
||||
|
||||
# With sep=";", columns should be separated by semicolon
|
||||
assert "id;name" in csv_data
|
||||
assert "1;Alice" in csv_data
|
||||
assert "2;Bob" in csv_data
|
||||
|
||||
|
||||
def test_csv_export_config_custom_decimal(mocker, mock_query) -> None:
|
||||
"""
|
||||
Test that streaming CSV export respects CSV_EXPORT config
|
||||
for custom decimal separator.
|
||||
|
||||
This is a regression test for GitHub issue #32371.
|
||||
"""
|
||||
mock_query.select_sql = "SELECT * FROM test"
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.keys.return_value = ["id", "price"]
|
||||
mock_result.fetchmany.side_effect = [
|
||||
[(1, 12.34), (2, 56.78)],
|
||||
[],
|
||||
]
|
||||
|
||||
mock_db, mock_session = _setup_sqllab_mocks(mocker, mock_query)
|
||||
|
||||
mock_connection = MagicMock()
|
||||
mock_connection.execution_options.return_value.execute.return_value = mock_result
|
||||
mock_connection.__enter__.return_value = mock_connection
|
||||
mock_connection.__exit__.return_value = None
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value = mock_connection
|
||||
mock_query.database.get_sqla_engine.return_value.__enter__.return_value = (
|
||||
mock_engine
|
||||
)
|
||||
|
||||
# Mock the app config to use comma as decimal separator
|
||||
mocker.patch.dict(
|
||||
"superset.commands.streaming_export.base.app.config",
|
||||
{"CSV_EXPORT": {"sep": ";", "decimal": ",", "encoding": "utf-8"}},
|
||||
)
|
||||
|
||||
command = StreamingSqlResultExportCommand("test_client_123")
|
||||
command.validate()
|
||||
|
||||
csv_generator_callable = command.run()
|
||||
generator = csv_generator_callable()
|
||||
csv_data = "".join(generator)
|
||||
|
||||
# With decimal=",", float values should use comma
|
||||
assert "12,34" in csv_data
|
||||
assert "56,78" in csv_data
|
||||
|
||||
|
||||
def test_csv_export_config_combined_sep_and_decimal(mocker, mock_query) -> None:
|
||||
"""
|
||||
Test that streaming CSV export respects both sep and decimal from CSV_EXPORT.
|
||||
|
||||
This is a regression test for GitHub issue #32371.
|
||||
"""
|
||||
mock_query.select_sql = "SELECT * FROM test"
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.keys.return_value = ["id", "name", "price"]
|
||||
mock_result.fetchmany.side_effect = [
|
||||
[(1, "Widget", 99.99), (2, "Gadget", 149.50)],
|
||||
[],
|
||||
]
|
||||
|
||||
mock_db, mock_session = _setup_sqllab_mocks(mocker, mock_query)
|
||||
|
||||
mock_connection = MagicMock()
|
||||
mock_connection.execution_options.return_value.execute.return_value = mock_result
|
||||
mock_connection.__enter__.return_value = mock_connection
|
||||
mock_connection.__exit__.return_value = None
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value = mock_connection
|
||||
mock_query.database.get_sqla_engine.return_value.__enter__.return_value = (
|
||||
mock_engine
|
||||
)
|
||||
|
||||
# Mock the app config to use European format
|
||||
mocker.patch.dict(
|
||||
"superset.commands.streaming_export.base.app.config",
|
||||
{"CSV_EXPORT": {"sep": ";", "decimal": ",", "encoding": "utf-8"}},
|
||||
)
|
||||
|
||||
command = StreamingSqlResultExportCommand("test_client_123")
|
||||
command.validate()
|
||||
|
||||
csv_generator_callable = command.run()
|
||||
generator = csv_generator_callable()
|
||||
csv_data = "".join(generator)
|
||||
|
||||
# Verify header uses semicolon separator
|
||||
assert "id;name;price" in csv_data
|
||||
# Verify data uses semicolon separator and comma decimal
|
||||
assert "1;Widget;99,99" in csv_data
|
||||
assert ";149,5" in csv_data
|
||||
|
||||
|
||||
def test_csv_export_config_custom_decimal_for_decimal_type(mocker, mock_query) -> None:
|
||||
"""
|
||||
Streaming CSV export must respect the custom decimal separator for
|
||||
``decimal.Decimal`` values too — SQLAlchemy commonly returns NUMERIC /
|
||||
DECIMAL columns as ``Decimal`` rather than ``float``.
|
||||
|
||||
Regression test for GitHub issue #32371 / PR #38170 review feedback.
|
||||
"""
|
||||
mock_query.select_sql = "SELECT * FROM test"
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.keys.return_value = ["id", "price"]
|
||||
mock_result.fetchmany.side_effect = [
|
||||
[(1, Decimal("12.34")), (2, Decimal("56.78"))],
|
||||
[],
|
||||
]
|
||||
|
||||
mock_db, mock_session = _setup_sqllab_mocks(mocker, mock_query)
|
||||
|
||||
mock_connection = MagicMock()
|
||||
mock_connection.execution_options.return_value.execute.return_value = mock_result
|
||||
mock_connection.__enter__.return_value = mock_connection
|
||||
mock_connection.__exit__.return_value = None
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value = mock_connection
|
||||
mock_query.database.get_sqla_engine.return_value.__enter__.return_value = (
|
||||
mock_engine
|
||||
)
|
||||
|
||||
mocker.patch.dict(
|
||||
"superset.commands.streaming_export.base.app.config",
|
||||
{"CSV_EXPORT": {"sep": ";", "decimal": ",", "encoding": "utf-8"}},
|
||||
)
|
||||
|
||||
command = StreamingSqlResultExportCommand("test_client_123")
|
||||
command.validate()
|
||||
|
||||
csv_generator_callable = command.run()
|
||||
generator = csv_generator_callable()
|
||||
csv_data = "".join(generator)
|
||||
|
||||
# Decimal values must be formatted with the custom separator, not left
|
||||
# with the default ``.`` which would slip through a ``float``-only check.
|
||||
assert "1;12,34" in csv_data
|
||||
assert "2;56,78" in csv_data
|
||||
assert "12.34" not in csv_data
|
||||
assert "56.78" not in csv_data
|
||||
|
||||
@@ -40,7 +40,7 @@ from superset.commands.database.uploaders.excel_reader import ExcelReader
|
||||
from superset.db_engine_specs.sqlite import SqliteEngineSpec
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
from superset.exceptions import OAuth2RedirectError, SupersetSecurityException
|
||||
from superset.sql.parse import Partition, Table
|
||||
from superset.sql.parse import Table
|
||||
from superset.superset_typing import OAuth2State
|
||||
from superset.utils import json
|
||||
from superset.utils.oauth2 import encode_oauth2_state
|
||||
@@ -1867,34 +1867,27 @@ def test_table_metadata_happy_path(
|
||||
Test the `table_metadata` endpoint.
|
||||
"""
|
||||
database = mocker.MagicMock()
|
||||
# Non-ODPS backend: partition detection short-circuits to (False, []).
|
||||
database.backend = "postgresql"
|
||||
database.db_engine_spec.get_table_metadata.return_value = {"hello": "world"}
|
||||
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
|
||||
mocker.patch("superset.databases.api.security_manager.raise_for_access")
|
||||
|
||||
no_partition = Partition(False, ())
|
||||
|
||||
response = client.get("/api/v1/database/1/table_metadata/?name=t")
|
||||
assert response.json == {"hello": "world"}
|
||||
database.db_engine_spec.get_table_metadata.assert_called_with(
|
||||
database,
|
||||
Table("t"),
|
||||
no_partition,
|
||||
)
|
||||
|
||||
response = client.get("/api/v1/database/1/table_metadata/?name=t&schema=s")
|
||||
database.db_engine_spec.get_table_metadata.assert_called_with(
|
||||
database,
|
||||
Table("t", "s"),
|
||||
no_partition,
|
||||
)
|
||||
|
||||
response = client.get("/api/v1/database/1/table_metadata/?name=t&catalog=c")
|
||||
database.db_engine_spec.get_table_metadata.assert_called_with(
|
||||
database,
|
||||
Table("t", None, "c"),
|
||||
no_partition,
|
||||
)
|
||||
|
||||
response = client.get(
|
||||
@@ -1903,7 +1896,6 @@ def test_table_metadata_happy_path(
|
||||
database.db_engine_spec.get_table_metadata.assert_called_with(
|
||||
database,
|
||||
Table("t", "s", "c"),
|
||||
no_partition,
|
||||
)
|
||||
|
||||
|
||||
@@ -1949,7 +1941,6 @@ def test_table_metadata_slashes(
|
||||
Test the `table_metadata` endpoint with names that have slashes.
|
||||
"""
|
||||
database = mocker.MagicMock()
|
||||
database.backend = "postgresql"
|
||||
database.db_engine_spec.get_table_metadata.return_value = {"hello": "world"}
|
||||
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
|
||||
mocker.patch("superset.databases.api.security_manager.raise_for_access")
|
||||
@@ -1958,7 +1949,6 @@ def test_table_metadata_slashes(
|
||||
database.db_engine_spec.get_table_metadata.assert_called_with(
|
||||
database,
|
||||
Table("foo/bar"),
|
||||
Partition(False, ()),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.dialects import sqlite
|
||||
|
||||
from superset.daos.database import DatabaseDAO
|
||||
from superset.db_engine_specs.odps import OdpsBaseEngineSpec, OdpsEngineSpec
|
||||
from superset.sql.parse import Partition, Table
|
||||
|
||||
|
||||
def test_odps_base_engine_spec_get_table_metadata_raises() -> None:
|
||||
"""OdpsBaseEngineSpec.get_table_metadata must not be called directly."""
|
||||
with pytest.raises(NotImplementedError):
|
||||
OdpsBaseEngineSpec.get_table_metadata(
|
||||
database=MagicMock(),
|
||||
table=Table("my_table", None, None),
|
||||
)
|
||||
|
||||
|
||||
def test_odps_engine_spec_select_star_no_partition() -> None:
|
||||
"""select_star for a non-partitioned ODPS table produces a plain SELECT *."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.get_columns.return_value = []
|
||||
database.compile_sqla_query = lambda query, catalog, schema: str(
|
||||
query.compile(dialect=sqlite.dialect())
|
||||
)
|
||||
dialect = sqlite.dialect()
|
||||
|
||||
sql = OdpsEngineSpec.select_star(
|
||||
database=database,
|
||||
table=Table("my_table", None, None),
|
||||
dialect=dialect,
|
||||
limit=100,
|
||||
show_cols=False,
|
||||
indent=False,
|
||||
latest_partition=False,
|
||||
partition=None,
|
||||
)
|
||||
|
||||
assert "SELECT" in sql
|
||||
assert "my_table" in sql
|
||||
|
||||
|
||||
def test_odps_engine_spec_select_star_with_partition() -> None:
|
||||
"""select_star for a partitioned ODPS table adds a WHERE clause."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.get_columns.return_value = []
|
||||
database.compile_sqla_query = lambda query, catalog, schema: str(
|
||||
query.compile(dialect=sqlite.dialect())
|
||||
)
|
||||
dialect = sqlite.dialect()
|
||||
partition = Partition(is_partitioned_table=True, partition_column=("month",))
|
||||
|
||||
sql = OdpsEngineSpec.select_star(
|
||||
database=database,
|
||||
table=Table("my_table", None, None),
|
||||
dialect=dialect,
|
||||
limit=100,
|
||||
show_cols=False,
|
||||
indent=False,
|
||||
latest_partition=False,
|
||||
partition=partition,
|
||||
)
|
||||
|
||||
assert "WHERE" in sql
|
||||
|
||||
|
||||
def test_is_odps_partitioned_table_non_odps_backend() -> None:
|
||||
"""Returns (False, []) immediately for non-ODPS databases; no network call made."""
|
||||
database = MagicMock()
|
||||
database.backend = "postgresql"
|
||||
|
||||
result = DatabaseDAO.is_odps_partitioned_table(database, "some_table")
|
||||
|
||||
assert result == (False, [])
|
||||
|
||||
|
||||
def test_is_odps_partitioned_table_missing_pyodps() -> None:
|
||||
"""Returns (False, []) with a warning when pyodps is not installed."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.sqlalchemy_uri = (
|
||||
"odps://mykey:mysecret@myproject/?endpoint=http://service.odps.test"
|
||||
)
|
||||
database.password = "mysecret" # noqa: S105
|
||||
|
||||
with patch("superset.daos.database.ODPS", None):
|
||||
result = DatabaseDAO.is_odps_partitioned_table(database, "some_table")
|
||||
|
||||
assert result == (False, [])
|
||||
|
||||
|
||||
def test_is_odps_partitioned_table_uri_no_match(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Logs a warning and returns (False, []) when the URI doesn't match the pattern."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.sqlalchemy_uri = "odps://invalid-uri-format"
|
||||
database.password = "secret" # noqa: S105
|
||||
|
||||
with patch("superset.daos.database.ODPS", MagicMock()):
|
||||
with caplog.at_level(logging.WARNING, logger="superset.daos.database"):
|
||||
result = DatabaseDAO.is_odps_partitioned_table(database, "some_table")
|
||||
|
||||
assert result == (False, [])
|
||||
assert "did not match" in caplog.text
|
||||
|
||||
|
||||
def test_is_odps_partitioned_table_partitioned(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Returns (True, [field_names]) for a partitioned ODPS table."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.sqlalchemy_uri = (
|
||||
"odps://mykey:mysecret@myproject/?endpoint=http://service.odps.test"
|
||||
)
|
||||
database.password = "mysecret" # noqa: S105
|
||||
|
||||
mock_partition = MagicMock()
|
||||
mock_partition.name = "month"
|
||||
mock_table = MagicMock()
|
||||
mock_table.exist_partition = True
|
||||
mock_table.table_schema.partitions = [mock_partition]
|
||||
|
||||
mock_odps_client = MagicMock()
|
||||
mock_odps_client.get_table.return_value = mock_table
|
||||
mock_odps_class = MagicMock(return_value=mock_odps_client)
|
||||
|
||||
with patch("superset.daos.database.ODPS", mock_odps_class):
|
||||
result = DatabaseDAO.is_odps_partitioned_table(database, "my_table")
|
||||
|
||||
assert result == (True, ["month"])
|
||||
|
||||
|
||||
def test_is_odps_partitioned_table_not_partitioned(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Returns (False, []) for a non-partitioned ODPS table."""
|
||||
database = MagicMock()
|
||||
database.backend = "odps"
|
||||
database.sqlalchemy_uri = (
|
||||
"odps://mykey:mysecret@myproject/?endpoint=http://service.odps.test"
|
||||
)
|
||||
database.password = "mysecret" # noqa: S105
|
||||
|
||||
mock_table = MagicMock()
|
||||
mock_table.exist_partition = False
|
||||
mock_odps_client = MagicMock()
|
||||
mock_odps_client.get_table.return_value = mock_table
|
||||
mock_odps_class = MagicMock(return_value=mock_odps_client)
|
||||
|
||||
with patch("superset.daos.database.ODPS", mock_odps_class):
|
||||
result = DatabaseDAO.is_odps_partitioned_table(database, "my_table")
|
||||
|
||||
assert result == (False, [])
|
||||
@@ -32,7 +32,6 @@ from superset.sql.parse import (
|
||||
KQLTokenType,
|
||||
KustoKQLStatement,
|
||||
LimitMethod,
|
||||
Partition,
|
||||
process_jinja_sql,
|
||||
remove_quotes,
|
||||
RLSMethod,
|
||||
@@ -140,41 +139,6 @@ def test_table_qualify() -> None:
|
||||
assert qualified.catalog == table.catalog
|
||||
|
||||
|
||||
def test_partition() -> None:
|
||||
"""
|
||||
Test the `Partition` class and its string conversion.
|
||||
"""
|
||||
# Test partitioned table with partition columns
|
||||
partition = Partition(is_partitioned_table=True, partition_column=("col1", "col2"))
|
||||
assert partition.is_partitioned_table is True
|
||||
assert partition.partition_column == ("col1", "col2")
|
||||
assert (
|
||||
str(partition)
|
||||
== "Partition(is_partitioned_table=True, partition_column=[col1, col2])"
|
||||
)
|
||||
|
||||
# Test non-partitioned table
|
||||
partition_none = Partition(is_partitioned_table=False, partition_column=None)
|
||||
assert partition_none.is_partitioned_table is False
|
||||
assert partition_none.partition_column is None
|
||||
assert (
|
||||
str(partition_none)
|
||||
== "Partition(is_partitioned_table=False, partition_column=[None])"
|
||||
)
|
||||
|
||||
# Test equality
|
||||
partition1 = Partition(is_partitioned_table=True, partition_column=("col1",))
|
||||
partition2 = Partition(is_partitioned_table=True, partition_column=("col1",))
|
||||
partition3 = Partition(is_partitioned_table=True, partition_column=("col2",))
|
||||
assert partition1 == partition2
|
||||
assert partition1 != partition3
|
||||
|
||||
# A frozen dataclass with a tuple field must be hashable (a list field would
|
||||
# raise TypeError: unhashable type at hash time).
|
||||
assert hash(partition1) == hash(partition2)
|
||||
assert len({partition1, partition2, partition3}) == 2
|
||||
|
||||
|
||||
def extract_tables_from_sql(sql: str, engine: str = "postgresql") -> set[Table]:
|
||||
"""
|
||||
Helper function to extract tables from SQL.
|
||||
|
||||
Reference in New Issue
Block a user