mirror of
https://github.com/apache/superset.git
synced 2026-07-27 17:12:36 +00:00
Compare commits
12 Commits
fix/deckgl
...
adopt-pr-2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31b4e5f0dd | ||
|
|
e0448ef3a4 | ||
|
|
41d89de139 | ||
|
|
d715176847 | ||
|
|
01ac966b83 | ||
|
|
97e5f0631d | ||
|
|
b7acb7984f | ||
|
|
d3919cf24f | ||
|
|
27889651b3 | ||
|
|
361fe6fe89 | ||
|
|
8506d70242 | ||
|
|
00a53eec2d |
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@@ -20,7 +20,7 @@
|
||||
|
||||
# Notify PMC members of changes to GitHub Actions
|
||||
|
||||
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @kgabryje @dpgaspar @sadpandajoe
|
||||
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @kgabryje @dpgaspar @sadpandajoe @hainenber
|
||||
|
||||
# Notify PMC members of changes to required GitHub Actions
|
||||
|
||||
|
||||
2
.github/workflows/ephemeral-env-pr-close.yml
vendored
2
.github/workflows/ephemeral-env-pr-close.yml
vendored
@@ -33,7 +33,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v5
|
||||
uses: aws-actions/configure-aws-credentials@v6
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
|
||||
4
.github/workflows/ephemeral-env.yml
vendored
4
.github/workflows/ephemeral-env.yml
vendored
@@ -189,7 +189,7 @@ jobs:
|
||||
--extra-flags "--build-arg INCLUDE_CHROMIUM=false"
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v5
|
||||
uses: aws-actions/configure-aws-credentials@v6
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
@@ -225,7 +225,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v5
|
||||
uses: aws-actions/configure-aws-credentials@v6
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
|
||||
@@ -80,6 +80,39 @@ instead requires a cachelib object.
|
||||
|
||||
See [Async Queries via Celery](/docs/configuration/async-queries-celery) for details.
|
||||
|
||||
## Celery beat
|
||||
|
||||
Superset has a Celery task that will periodically warm up the cache based on different strategies.
|
||||
To use it, add the following to your `superset_config.py`:
|
||||
|
||||
```python
|
||||
from celery.schedules import crontab
|
||||
from superset.config import CeleryConfig
|
||||
|
||||
# User that will be used to authenticate and render dashboards for cache warmup
|
||||
SUPERSET_CACHE_WARMUP_USER = "user_with_permission_to_dashboards"
|
||||
|
||||
# Extend the default CeleryConfig to add cache warmup schedule
|
||||
class CustomCeleryConfig(CeleryConfig):
|
||||
beat_schedule = {
|
||||
**CeleryConfig.beat_schedule,
|
||||
'cache-warmup-hourly': {
|
||||
'task': 'cache-warmup',
|
||||
'schedule': crontab(minute=0, hour='*'), # hourly
|
||||
'kwargs': {
|
||||
'strategy_name': 'top_n_dashboards',
|
||||
'top_n': 5,
|
||||
'since': '7 days ago',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
CELERY_CONFIG = CustomCeleryConfig
|
||||
```
|
||||
|
||||
This will cache the top 5 most popular dashboards every hour. For other
|
||||
strategies, check the `superset/tasks/cache.py` file.
|
||||
|
||||
## Caching Thumbnails
|
||||
|
||||
This is an optional feature that can be turned on by activating its [feature flag](/docs/configuration/configuring-superset#feature-flags) on config:
|
||||
|
||||
1114
superset-embedded-sdk/package-lock.json
generated
1114
superset-embedded-sdk/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
44
superset-frontend/package-lock.json
generated
44
superset-frontend/package-lock.json
generated
@@ -89,7 +89,7 @@
|
||||
"json-stringify-pretty-compact": "^2.0.0",
|
||||
"lodash": "^4.17.23",
|
||||
"mapbox-gl": "^3.18.1",
|
||||
"markdown-to-jsx": "^9.6.1",
|
||||
"markdown-to-jsx": "^9.7.2",
|
||||
"match-sorter": "^6.3.4",
|
||||
"memoize-one": "^5.2.1",
|
||||
"mousetrap": "^1.6.5",
|
||||
@@ -144,7 +144,7 @@
|
||||
"@applitools/eyes-storybook": "^3.63.10",
|
||||
"@babel/cli": "^7.28.6",
|
||||
"@babel/compat-data": "^7.28.4",
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/eslint-parser": "^7.28.6",
|
||||
"@babel/node": "^7.28.6",
|
||||
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
|
||||
@@ -1184,21 +1184,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/core": {
|
||||
"version": "7.28.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.6.tgz",
|
||||
"integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==",
|
||||
"version": "7.29.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz",
|
||||
"integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.28.6",
|
||||
"@babel/generator": "^7.28.6",
|
||||
"@babel/code-frame": "^7.29.0",
|
||||
"@babel/generator": "^7.29.0",
|
||||
"@babel/helper-compilation-targets": "^7.28.6",
|
||||
"@babel/helper-module-transforms": "^7.28.6",
|
||||
"@babel/helpers": "^7.28.6",
|
||||
"@babel/parser": "^7.28.6",
|
||||
"@babel/parser": "^7.29.0",
|
||||
"@babel/template": "^7.28.6",
|
||||
"@babel/traverse": "^7.28.6",
|
||||
"@babel/types": "^7.28.6",
|
||||
"@babel/traverse": "^7.29.0",
|
||||
"@babel/types": "^7.29.0",
|
||||
"@jridgewell/remapping": "^2.3.5",
|
||||
"convert-source-map": "^2.0.0",
|
||||
"debug": "^4.1.0",
|
||||
@@ -43189,9 +43189,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/markdown-to-jsx": {
|
||||
"version": "9.6.1",
|
||||
"resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-9.6.1.tgz",
|
||||
"integrity": "sha512-FLZPygHQzEGNUwIJ3Egdf3Lzm5M4ebswji8aMfM5tAq1vxkSw31a7OBmYXA3tg264PPyJw5UMDHw1fU+wFaQ9Q==",
|
||||
"version": "9.7.2",
|
||||
"resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-9.7.2.tgz",
|
||||
"integrity": "sha512-DTWRWPPyTVww//5s/dZgq1Pl6i5CclnKvx847Hpj6/GQDpYzM1xkD/Z84PzaVPk/zWfEdJKZSr2G/Y7RTYw7Fw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
@@ -61020,7 +61020,7 @@
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.28.6",
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/preset-env": "^7.29.0",
|
||||
"@babel/preset-react": "^7.28.5",
|
||||
"@babel/preset-typescript": "^7.28.5",
|
||||
@@ -64998,7 +64998,7 @@
|
||||
"react-resizable": "^3.1.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/preset-env": "^7.29.0",
|
||||
"@babel/preset-react": "^7.28.5",
|
||||
"@babel/preset-typescript": "^7.28.5",
|
||||
@@ -66400,20 +66400,6 @@
|
||||
"react-dom": "^17.0.2"
|
||||
}
|
||||
},
|
||||
"plugins/plugin-chart-pivot-table/node_modules/@babel/types": {
|
||||
"version": "7.29.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz",
|
||||
"integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-string-parser": "^7.27.1",
|
||||
"@babel/helper-validator-identifier": "^7.28.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"plugins/plugin-chart-table": {
|
||||
"name": "@superset-ui/plugin-chart-table",
|
||||
"version": "0.20.3",
|
||||
|
||||
@@ -171,7 +171,7 @@
|
||||
"json-stringify-pretty-compact": "^2.0.0",
|
||||
"lodash": "^4.17.23",
|
||||
"mapbox-gl": "^3.18.1",
|
||||
"markdown-to-jsx": "^9.6.1",
|
||||
"markdown-to-jsx": "^9.7.2",
|
||||
"match-sorter": "^6.3.4",
|
||||
"memoize-one": "^5.2.1",
|
||||
"mousetrap": "^1.6.5",
|
||||
@@ -226,7 +226,7 @@
|
||||
"@applitools/eyes-storybook": "^3.63.10",
|
||||
"@babel/cli": "^7.28.6",
|
||||
"@babel/compat-data": "^7.28.4",
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/eslint-parser": "^7.28.6",
|
||||
"@babel/node": "^7.28.6",
|
||||
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.28.6",
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/preset-env": "^7.29.0",
|
||||
"@babel/preset-react": "^7.28.5",
|
||||
"@babel/preset-typescript": "^7.28.5",
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
"react-resizable": "^3.1.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.28.6",
|
||||
"@babel/core": "^7.29.0",
|
||||
"@babel/preset-env": "^7.29.0",
|
||||
"@babel/preset-react": "^7.28.5",
|
||||
"@babel/preset-typescript": "^7.28.5",
|
||||
|
||||
@@ -35,7 +35,7 @@ import {
|
||||
getStandardizedControls,
|
||||
} from '@superset-ui/chart-controls';
|
||||
import { DEFAULT_FORM_DATA } from './types';
|
||||
import { LABEL_POSITION } from '../constants';
|
||||
import { LabelPositionEnum } from '../types';
|
||||
import { legendSection } from '../controls';
|
||||
|
||||
const { labelType, labelPosition, numberFormat, showLabels, isCircle } =
|
||||
@@ -72,6 +72,22 @@ const radarMetricMinValue: { name: string; config: ControlFormItemSpec } = {
|
||||
},
|
||||
};
|
||||
|
||||
const getLabelPositionOptions = (): [LabelPositionEnum, string][] => [
|
||||
[LabelPositionEnum.Top, t('Top')],
|
||||
[LabelPositionEnum.Left, t('Left')],
|
||||
[LabelPositionEnum.Right, t('Right')],
|
||||
[LabelPositionEnum.Bottom, t('Bottom')],
|
||||
[LabelPositionEnum.Inside, t('Inside')],
|
||||
[LabelPositionEnum.InsideLeft, t('Inside left')],
|
||||
[LabelPositionEnum.InsideRight, t('Inside right')],
|
||||
[LabelPositionEnum.InsideTop, t('Inside top')],
|
||||
[LabelPositionEnum.InsideBottom, t('Inside bottom')],
|
||||
[LabelPositionEnum.InsideTopLeft, t('Inside top left')],
|
||||
[LabelPositionEnum.InsideBottomLeft, t('Inside bottom left')],
|
||||
[LabelPositionEnum.InsideTopRight, t('Inside top right')],
|
||||
[LabelPositionEnum.InsideBottomRight, t('Inside bottom right')],
|
||||
];
|
||||
|
||||
const config: ControlPanelConfig = {
|
||||
controlPanelSections: [
|
||||
{
|
||||
@@ -136,7 +152,7 @@ const config: ControlPanelConfig = {
|
||||
freeForm: false,
|
||||
label: t('Label position'),
|
||||
renderTrigger: true,
|
||||
choices: LABEL_POSITION,
|
||||
choices: getLabelPositionOptions(),
|
||||
default: labelPosition,
|
||||
description: D3_FORMAT_DOCS,
|
||||
},
|
||||
|
||||
@@ -21,7 +21,6 @@ import { t } from '@apache-superset/core';
|
||||
import { JsonValue, TimeGranularity } from '@superset-ui/core';
|
||||
import { ReactNode } from 'react';
|
||||
import {
|
||||
LabelPositionEnum,
|
||||
LegendFormData,
|
||||
LegendOrientation,
|
||||
LegendType,
|
||||
@@ -50,22 +49,6 @@ export const TIMESERIES_CONSTANTS = {
|
||||
horizontalBarLabelRightPadding: 70,
|
||||
};
|
||||
|
||||
export const LABEL_POSITION: [LabelPositionEnum, string][] = [
|
||||
[LabelPositionEnum.Top, 'Top'],
|
||||
[LabelPositionEnum.Left, 'Left'],
|
||||
[LabelPositionEnum.Right, 'Right'],
|
||||
[LabelPositionEnum.Bottom, 'Bottom'],
|
||||
[LabelPositionEnum.Inside, 'Inside'],
|
||||
[LabelPositionEnum.InsideLeft, 'Inside left'],
|
||||
[LabelPositionEnum.InsideRight, 'Inside right'],
|
||||
[LabelPositionEnum.InsideTop, 'Inside top'],
|
||||
[LabelPositionEnum.InsideBottom, 'Inside bottom'],
|
||||
[LabelPositionEnum.InsideTopLeft, 'Inside top left'],
|
||||
[LabelPositionEnum.InsideBottomLeft, 'Inside bottom left'],
|
||||
[LabelPositionEnum.InsideTopRight, 'Inside top right'],
|
||||
[LabelPositionEnum.InsideBottomRight, 'Inside bottom right'],
|
||||
];
|
||||
|
||||
export enum OpacityEnum {
|
||||
Transparent = 0,
|
||||
SemiTransparent = 0.3,
|
||||
|
||||
@@ -1047,6 +1047,9 @@ THUMBNAIL_CACHE_CONFIG: CacheConfig = {
|
||||
}
|
||||
THUMBNAIL_ERROR_CACHE_TTL = int(timedelta(days=1).total_seconds())
|
||||
|
||||
# Cache warmup user
|
||||
SUPERSET_CACHE_WARMUP_USER = "admin"
|
||||
|
||||
# Time before selenium times out after trying to locate an element on the page and wait
|
||||
# for that element to load for a screenshot.
|
||||
SCREENSHOT_LOCATE_WAIT = int(timedelta(seconds=10).total_seconds())
|
||||
|
||||
@@ -21,8 +21,6 @@ Pydantic schemas for chart-related responses
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any, Dict, List, Literal, Protocol
|
||||
|
||||
@@ -50,6 +48,10 @@ from superset.mcp_service.system.schemas import (
|
||||
TagInfo,
|
||||
UserInfo,
|
||||
)
|
||||
from superset.mcp_service.utils.sanitization import (
|
||||
sanitize_filter_value,
|
||||
sanitize_user_input,
|
||||
)
|
||||
|
||||
|
||||
class ChartLike(Protocol):
|
||||
@@ -357,113 +359,17 @@ class ColumnRef(BaseModel):
|
||||
@classmethod
|
||||
def sanitize_name(cls, v: str) -> str:
|
||||
"""Sanitize column name to prevent XSS and SQL injection."""
|
||||
if not v or not v.strip():
|
||||
raise ValueError("Column name cannot be empty")
|
||||
|
||||
# Length check first to prevent ReDoS attacks
|
||||
if len(v) > 255:
|
||||
raise ValueError(
|
||||
f"Column name too long ({len(v)} characters). "
|
||||
f"Maximum allowed length is 255 characters."
|
||||
)
|
||||
|
||||
# Remove HTML tags and decode entities
|
||||
sanitized = html.escape(v.strip())
|
||||
|
||||
# Check for dangerous HTML tags using substring checks (safe)
|
||||
dangerous_tags = ["<script", "</script>", "<iframe", "<object", "<embed"]
|
||||
v_lower = v.lower()
|
||||
for tag in dangerous_tags:
|
||||
if tag in v_lower:
|
||||
raise ValueError(
|
||||
"Column name contains potentially malicious script content"
|
||||
)
|
||||
|
||||
# Check URL schemes with word boundaries to match only actual URLs
|
||||
if re.search(r"\b(javascript|vbscript|data):", v, re.IGNORECASE):
|
||||
raise ValueError("Column name contains potentially malicious URL scheme")
|
||||
|
||||
# Basic SQL injection patterns (basic protection)
|
||||
# Use simple patterns without backtracking
|
||||
dangerous_patterns = [
|
||||
r"[;|&$`]", # Dangerous shell characters
|
||||
r"\b(DROP|DELETE|INSERT|UPDATE|CREATE|ALTER|EXEC|EXECUTE)\b",
|
||||
r"--", # SQL comment
|
||||
r"/\*", # SQL comment start (just check for start, not full pattern)
|
||||
]
|
||||
|
||||
for pattern in dangerous_patterns:
|
||||
if re.search(pattern, v, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Column name contains potentially unsafe characters or SQL keywords"
|
||||
)
|
||||
|
||||
return sanitized
|
||||
# sanitize_user_input raises ValueError when allow_empty=False (default)
|
||||
# so the return value is guaranteed to be a non-None str
|
||||
return sanitize_user_input(
|
||||
v, "Column name", max_length=255, check_sql_keywords=True
|
||||
) # type: ignore[return-value]
|
||||
|
||||
@field_validator("label")
|
||||
@classmethod
|
||||
def sanitize_label(cls, v: str | None) -> str | None:
|
||||
"""Sanitize display label to prevent XSS attacks."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Strip whitespace
|
||||
v = v.strip()
|
||||
if not v:
|
||||
return None
|
||||
|
||||
# Length check first to prevent ReDoS attacks
|
||||
if len(v) > 500:
|
||||
raise ValueError(
|
||||
f"Label too long ({len(v)} characters). "
|
||||
f"Maximum allowed length is 500 characters."
|
||||
)
|
||||
|
||||
# Check for dangerous HTML tags and JavaScript protocols using substring checks
|
||||
# This avoids ReDoS vulnerabilities from regex patterns
|
||||
dangerous_tags = [
|
||||
"<script",
|
||||
"</script>",
|
||||
"<iframe",
|
||||
"</iframe>",
|
||||
"<object",
|
||||
"</object>",
|
||||
"<embed",
|
||||
"</embed>",
|
||||
"<link",
|
||||
"<meta",
|
||||
]
|
||||
|
||||
v_lower = v.lower()
|
||||
for tag in dangerous_tags:
|
||||
if tag in v_lower:
|
||||
raise ValueError(
|
||||
"Label contains potentially malicious content. "
|
||||
"HTML tags, JavaScript, and event handlers are not allowed "
|
||||
"in labels."
|
||||
)
|
||||
|
||||
# Check URL schemes and event handlers with word boundaries
|
||||
dangerous_patterns = [
|
||||
r"\b(javascript|vbscript|data):", # URL schemes
|
||||
r"on\w+\s*=", # Event handlers
|
||||
]
|
||||
for pattern in dangerous_patterns:
|
||||
if re.search(pattern, v, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Label contains potentially malicious content. "
|
||||
"HTML tags, JavaScript, and event handlers are not allowed."
|
||||
)
|
||||
|
||||
# Filter dangerous Unicode characters
|
||||
v = re.sub(
|
||||
r"[\u200B-\u200D\uFEFF\u0000-\u0008\u000B\u000C\u000E-\u001F]", "", v
|
||||
)
|
||||
|
||||
# HTML escape the cleaned content
|
||||
sanitized = html.escape(v)
|
||||
|
||||
return sanitized if sanitized else None
|
||||
return sanitize_user_input(v, "Label", max_length=500, allow_empty=True)
|
||||
|
||||
|
||||
class AxisConfig(BaseModel):
|
||||
@@ -496,112 +402,15 @@ class FilterConfig(BaseModel):
|
||||
@classmethod
|
||||
def sanitize_column(cls, v: str) -> str:
|
||||
"""Sanitize filter column name to prevent injection attacks."""
|
||||
if not v or not v.strip():
|
||||
raise ValueError("Filter column name cannot be empty")
|
||||
|
||||
# Length check first to prevent ReDoS attacks
|
||||
if len(v) > 255:
|
||||
raise ValueError(
|
||||
f"Filter column name too long ({len(v)} characters). "
|
||||
f"Maximum allowed length is 255 characters."
|
||||
)
|
||||
|
||||
# Remove HTML tags and decode entities
|
||||
sanitized = html.escape(v.strip())
|
||||
|
||||
# Check for dangerous HTML tags using substring checks (safe)
|
||||
dangerous_tags = ["<script", "</script>"]
|
||||
v_lower = v.lower()
|
||||
for tag in dangerous_tags:
|
||||
if tag in v_lower:
|
||||
raise ValueError(
|
||||
"Filter column contains potentially malicious script content"
|
||||
)
|
||||
|
||||
# Check URL schemes with word boundaries
|
||||
if re.search(r"\b(javascript|vbscript|data):", v, re.IGNORECASE):
|
||||
raise ValueError("Filter column contains potentially malicious URL scheme")
|
||||
|
||||
return sanitized
|
||||
|
||||
@staticmethod
|
||||
def _validate_string_value(v: str) -> None:
|
||||
"""Validate string filter value for security issues."""
|
||||
# Check for dangerous HTML tags and SQL procedures
|
||||
dangerous_substrings = [
|
||||
"<script",
|
||||
"</script>",
|
||||
"<iframe",
|
||||
"<object",
|
||||
"<embed",
|
||||
"xp_cmdshell",
|
||||
"sp_executesql",
|
||||
]
|
||||
v_lower = v.lower()
|
||||
for substring in dangerous_substrings:
|
||||
if substring in v_lower:
|
||||
raise ValueError(
|
||||
"Filter value contains potentially malicious content. "
|
||||
"HTML tags and JavaScript are not allowed."
|
||||
)
|
||||
|
||||
# Check URL schemes with word boundaries
|
||||
if re.search(r"\b(javascript|vbscript|data):", v, re.IGNORECASE):
|
||||
raise ValueError("Filter value contains potentially malicious URL scheme")
|
||||
|
||||
# SQL injection patterns
|
||||
sql_patterns = [
|
||||
r";\s*(DROP|DELETE|INSERT|UPDATE|CREATE|ALTER|EXEC|EXECUTE)\b",
|
||||
r"'\s*OR\s*'",
|
||||
r"'\s*AND\s*'",
|
||||
r"--\s*",
|
||||
r"/\*",
|
||||
r"UNION\s+SELECT",
|
||||
]
|
||||
for pattern in sql_patterns:
|
||||
if re.search(pattern, v, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Filter value contains potentially malicious SQL patterns."
|
||||
)
|
||||
|
||||
# Check for other dangerous patterns
|
||||
if re.search(r"[;&|`$()]", v):
|
||||
raise ValueError(
|
||||
"Filter value contains potentially unsafe shell characters."
|
||||
)
|
||||
if re.search(r"on\w+\s*=", v, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Filter value contains potentially malicious event handlers."
|
||||
)
|
||||
if re.search(r"\\x[0-9a-fA-F]{2}", v):
|
||||
raise ValueError("Filter value contains hex encoding which is not allowed.")
|
||||
# sanitize_user_input raises ValueError when allow_empty=False (default)
|
||||
# so the return value is guaranteed to be a non-None str
|
||||
return sanitize_user_input(v, "Filter column", max_length=255) # type: ignore[return-value]
|
||||
|
||||
@field_validator("value")
|
||||
@classmethod
|
||||
def sanitize_value(cls, v: str | int | float | bool) -> str | int | float | bool:
|
||||
"""Sanitize filter value to prevent XSS and SQL injection attacks."""
|
||||
if isinstance(v, str):
|
||||
v = v.strip()
|
||||
|
||||
# Length check FIRST to prevent ReDoS attacks
|
||||
if len(v) > 1000:
|
||||
raise ValueError(
|
||||
f"Filter value too long ({len(v)} characters). "
|
||||
f"Maximum allowed length is 1000 characters."
|
||||
)
|
||||
|
||||
# Validate security
|
||||
cls._validate_string_value(v)
|
||||
|
||||
# Filter dangerous Unicode characters
|
||||
v = re.sub(
|
||||
r"[\u200B-\u200D\uFEFF\u0000-\u0008\u000B\u000C\u000E-\u001F]", "", v
|
||||
)
|
||||
|
||||
# HTML escape the cleaned content
|
||||
return html.escape(v)
|
||||
|
||||
return v # Return non-string values as-is
|
||||
return sanitize_filter_value(v, max_length=1000)
|
||||
|
||||
|
||||
# Actual chart types
|
||||
@@ -848,6 +657,11 @@ class ListChartsRequest(MetadataCacheControl):
|
||||
class GenerateChartRequest(QueryCacheControl):
|
||||
dataset_id: int | str = Field(..., description="Dataset identifier (ID, UUID)")
|
||||
config: ChartConfig = Field(..., description="Chart configuration")
|
||||
chart_name: str | None = Field(
|
||||
None,
|
||||
description="Custom chart name (optional, auto-generates if not provided)",
|
||||
max_length=255,
|
||||
)
|
||||
save_chart: bool = Field(
|
||||
default=False,
|
||||
description="Whether to permanently save the chart in Superset",
|
||||
@@ -861,6 +675,12 @@ class GenerateChartRequest(QueryCacheControl):
|
||||
description="List of preview formats to generate",
|
||||
)
|
||||
|
||||
@field_validator("chart_name")
|
||||
@classmethod
|
||||
def sanitize_chart_name(cls, v: str | None) -> str | None:
|
||||
"""Sanitize chart name to prevent XSS attacks."""
|
||||
return sanitize_user_input(v, "Chart name", max_length=255, allow_empty=True)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_cache_timeout(self) -> "GenerateChartRequest":
|
||||
"""Validate cache timeout is non-negative."""
|
||||
@@ -911,62 +731,7 @@ class UpdateChartRequest(QueryCacheControl):
|
||||
@classmethod
|
||||
def sanitize_chart_name(cls, v: str | None) -> str | None:
|
||||
"""Sanitize chart name to prevent XSS attacks."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Strip whitespace
|
||||
v = v.strip()
|
||||
if not v:
|
||||
return None
|
||||
|
||||
# Length check first to prevent ReDoS attacks
|
||||
if len(v) > 255:
|
||||
raise ValueError(
|
||||
f"Chart name too long ({len(v)} characters). "
|
||||
f"Maximum allowed length is 255 characters."
|
||||
)
|
||||
|
||||
# Check for dangerous HTML tags using substring checks (safe)
|
||||
dangerous_tags = [
|
||||
"<script",
|
||||
"</script>",
|
||||
"<iframe",
|
||||
"</iframe>",
|
||||
"<object",
|
||||
"</object>",
|
||||
"<embed",
|
||||
"</embed>",
|
||||
"<link",
|
||||
"<meta",
|
||||
]
|
||||
|
||||
v_lower = v.lower()
|
||||
for tag in dangerous_tags:
|
||||
if tag in v_lower:
|
||||
raise ValueError(
|
||||
"Chart name contains potentially malicious content. "
|
||||
"HTML tags and JavaScript are not allowed in chart names."
|
||||
)
|
||||
|
||||
# Check URL schemes with word boundaries
|
||||
if re.search(r"\b(javascript|vbscript|data):", v, re.IGNORECASE):
|
||||
raise ValueError("Chart name contains potentially malicious URL scheme")
|
||||
|
||||
# Check for event handlers with simple regex
|
||||
if re.search(r"on\w+\s*=", v, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Chart name contains potentially malicious event handlers."
|
||||
)
|
||||
|
||||
# Filter dangerous Unicode characters
|
||||
v = re.sub(
|
||||
r"[\u200B-\u200D\uFEFF\u0000-\u0008\u000B\u000C\u000E-\u001F]", "", v
|
||||
)
|
||||
|
||||
# HTML escape the cleaned content
|
||||
sanitized = html.escape(v)
|
||||
|
||||
return sanitized if sanitized else None
|
||||
return sanitize_user_input(v, "Chart name", max_length=255, allow_empty=True)
|
||||
|
||||
|
||||
class UpdateChartPreviewRequest(FormDataCacheControl):
|
||||
|
||||
@@ -189,9 +189,9 @@ async def generate_chart( # noqa: C901
|
||||
await ctx.report_progress(2, 5, "Creating chart in database")
|
||||
from superset.commands.chart.create import CreateChartCommand
|
||||
|
||||
# Generate a chart name
|
||||
chart_name = generate_chart_name(request.config)
|
||||
await ctx.debug("Generated chart name: chart_name=%s" % (chart_name,))
|
||||
# Use custom chart name if provided, otherwise auto-generate
|
||||
chart_name = request.chart_name or generate_chart_name(request.config)
|
||||
await ctx.debug("Chart name: chart_name=%s" % (chart_name,))
|
||||
|
||||
# Find the dataset to get its numeric ID
|
||||
from superset.daos.dataset import DatasetDAO
|
||||
|
||||
283
superset/mcp_service/utils/sanitization.py
Normal file
283
superset/mcp_service/utils/sanitization.py
Normal file
@@ -0,0 +1,283 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Centralized sanitization utilities for MCP service user inputs.
|
||||
|
||||
This module uses the nh3 library (Rust-based HTML sanitizer) to strip malicious
|
||||
HTML tags and protocols from user inputs. nh3 is faster and safer than manual
|
||||
regex-based sanitization.
|
||||
|
||||
Key features:
|
||||
- Strips all HTML tags using nh3.clean() with no allowed tags
|
||||
- Blocks dangerous URL schemes (javascript:, vbscript:, data:)
|
||||
- Preserves safe text content (e.g., '&' stays as '&', not '&')
|
||||
- Additional SQL injection protection for database-facing inputs
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
import nh3
|
||||
|
||||
|
||||
def _strip_html_tags(value: str) -> str:
|
||||
"""
|
||||
Strip all HTML tags from the input using nh3.
|
||||
|
||||
Decodes all layers of HTML entity encoding BEFORE passing to nh3,
|
||||
so entity-encoded tags (e.g., ``<script>``) are decoded into
|
||||
real tags that nh3 can detect and strip. After nh3 removes all tags,
|
||||
we only restore ``&`` back to ``&`` (not a full html.unescape)
|
||||
to preserve ampersands in display text without risking XSS from
|
||||
re-introducing angle brackets or other HTML-significant characters.
|
||||
|
||||
Args:
|
||||
value: The input string that may contain HTML
|
||||
|
||||
Returns:
|
||||
String with all HTML tags removed and ampersands preserved
|
||||
"""
|
||||
# Decode all layers of HTML entity encoding to prevent bypass
|
||||
# via entity-encoded tags (e.g., <script> or &lt;script&gt;)
|
||||
# The loop terminates when unescape produces no change (idempotent on decoded text).
|
||||
# Max iterations cap provides defense-in-depth against pathological inputs.
|
||||
max_iterations = 100
|
||||
decoded = value
|
||||
prev = None
|
||||
iterations = 0
|
||||
while prev != decoded and iterations < max_iterations:
|
||||
prev = decoded
|
||||
decoded = html.unescape(decoded)
|
||||
iterations += 1
|
||||
|
||||
# nh3.clean with tags=set() strips ALL HTML tags from the decoded input
|
||||
# url_schemes=set() blocks all URL schemes in any remaining attributes
|
||||
cleaned = nh3.clean(decoded, tags=set(), url_schemes=set())
|
||||
|
||||
# Only restore & → & to preserve ampersands in display text (e.g. "A & B").
|
||||
# Do NOT use html.unescape() here: nh3 may pass through HTML entities from
|
||||
# the input (e.g. <script>), and a full unescape would re-introduce
|
||||
# raw angle brackets, creating an XSS vector.
|
||||
return cleaned.replace("&", "&")
|
||||
|
||||
|
||||
def _check_dangerous_patterns(value: str, field_name: str) -> None:
|
||||
"""
|
||||
Check for dangerous patterns that nh3 doesn't catch.
|
||||
|
||||
This includes URL schemes in plain text (not in HTML attributes),
|
||||
event handler patterns, and dangerous Unicode characters.
|
||||
|
||||
Args:
|
||||
value: The input string to check
|
||||
field_name: Name of the field (for error messages)
|
||||
|
||||
Raises:
|
||||
ValueError: If dangerous patterns are found
|
||||
"""
|
||||
# Block dangerous URL schemes in plain text (word boundary check)
|
||||
if re.search(r"\b(javascript|vbscript|data):", value, re.IGNORECASE):
|
||||
raise ValueError(f"{field_name} contains potentially malicious URL scheme")
|
||||
|
||||
# Block event handler patterns (onclick=, onerror=, etc.)
|
||||
if re.search(r"on\w+\s*=", value, re.IGNORECASE):
|
||||
raise ValueError(f"{field_name} contains potentially malicious event handler")
|
||||
|
||||
|
||||
def _check_sql_patterns(value: str, field_name: str) -> None:
|
||||
"""
|
||||
Check for SQL injection patterns.
|
||||
|
||||
Args:
|
||||
value: The input string to check
|
||||
field_name: Name of the field (for error messages)
|
||||
|
||||
Raises:
|
||||
ValueError: If SQL injection patterns are found
|
||||
"""
|
||||
# Check for dangerous SQL keywords
|
||||
if re.search(
|
||||
r"\b(DROP|DELETE|INSERT|UPDATE|CREATE|ALTER|EXEC|EXECUTE)\b",
|
||||
value,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
raise ValueError(f"{field_name} contains potentially unsafe SQL keywords")
|
||||
|
||||
# Check for shell metacharacters and SQL comments
|
||||
if re.search(r"[;|&$`]|--", value):
|
||||
raise ValueError(f"{field_name} contains potentially unsafe characters")
|
||||
|
||||
# Check for SQL comment start
|
||||
if "/*" in value:
|
||||
raise ValueError(f"{field_name} contains potentially unsafe SQL comment syntax")
|
||||
|
||||
|
||||
def _remove_dangerous_unicode(value: str) -> str:
|
||||
"""
|
||||
Remove dangerous Unicode characters (zero-width, control chars).
|
||||
|
||||
Args:
|
||||
value: The input string
|
||||
|
||||
Returns:
|
||||
String with dangerous Unicode characters removed
|
||||
"""
|
||||
return re.sub(
|
||||
r"[\u200B-\u200D\uFEFF\u0000-\u0008\u000B\u000C\u000E-\u001F]", "", value
|
||||
)
|
||||
|
||||
|
||||
def sanitize_user_input(
|
||||
value: str | None,
|
||||
field_name: str,
|
||||
max_length: int = 255,
|
||||
check_sql_keywords: bool = False,
|
||||
allow_empty: bool = False,
|
||||
) -> str | None:
|
||||
"""
|
||||
Centralized sanitization for user-provided text inputs.
|
||||
|
||||
Uses nh3 to strip HTML tags and performs additional security checks.
|
||||
|
||||
Args:
|
||||
value: The input string to sanitize
|
||||
field_name: Name of the field (for error messages)
|
||||
max_length: Maximum allowed length
|
||||
check_sql_keywords: Whether to check for SQL injection keywords
|
||||
allow_empty: Whether to allow empty/None values
|
||||
|
||||
Returns:
|
||||
Sanitized string, or None if allow_empty=True and value is empty
|
||||
|
||||
Raises:
|
||||
ValueError: If value fails security validation
|
||||
|
||||
Security checks performed:
|
||||
- Strips all HTML tags using nh3 (Rust-based sanitizer)
|
||||
- Blocks JavaScript/VBScript/data URL schemes
|
||||
- Blocks event handlers (onclick=, onerror=, etc.)
|
||||
- Removes dangerous Unicode characters (zero-width, control chars)
|
||||
- SQL keywords and shell metacharacters (when check_sql_keywords=True)
|
||||
"""
|
||||
if value is None:
|
||||
if allow_empty:
|
||||
return None
|
||||
raise ValueError(f"{field_name} cannot be empty")
|
||||
|
||||
value = value.strip()
|
||||
|
||||
if not value:
|
||||
if allow_empty:
|
||||
return None
|
||||
raise ValueError(f"{field_name} cannot be empty")
|
||||
|
||||
# Length check first to prevent ReDoS attacks
|
||||
if len(value) > max_length:
|
||||
raise ValueError(
|
||||
f"{field_name} too long ({len(value)} characters). "
|
||||
f"Maximum allowed length is {max_length} characters."
|
||||
)
|
||||
|
||||
# Strip all HTML tags using nh3
|
||||
value = _strip_html_tags(value)
|
||||
|
||||
# Check for dangerous patterns (URL schemes, event handlers)
|
||||
_check_dangerous_patterns(value, field_name)
|
||||
|
||||
# SQL keyword and shell metacharacter checks (for column names, etc.)
|
||||
if check_sql_keywords:
|
||||
_check_sql_patterns(value, field_name)
|
||||
|
||||
# Remove dangerous Unicode characters
|
||||
value = _remove_dangerous_unicode(value)
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def sanitize_filter_value(
|
||||
value: str | int | float | bool,
|
||||
max_length: int = 1000,
|
||||
) -> str | int | float | bool:
|
||||
"""
|
||||
Sanitize filter values which can be strings or other types.
|
||||
|
||||
For non-string values, returns as-is (no sanitization needed).
|
||||
For strings, uses nh3 to strip HTML and applies security validation.
|
||||
|
||||
Args:
|
||||
value: The filter value (string, int, float, or bool)
|
||||
max_length: Maximum length for string values
|
||||
|
||||
Returns:
|
||||
Sanitized value
|
||||
|
||||
Raises:
|
||||
ValueError: If string value fails security validation
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
|
||||
value = value.strip()
|
||||
|
||||
# Length check first
|
||||
if len(value) > max_length:
|
||||
raise ValueError(
|
||||
f"Filter value too long ({len(value)} characters). "
|
||||
f"Maximum allowed length is {max_length} characters."
|
||||
)
|
||||
|
||||
# Strip all HTML tags using nh3
|
||||
value = _strip_html_tags(value)
|
||||
|
||||
# Check for dangerous patterns
|
||||
_check_dangerous_patterns(value, "Filter value")
|
||||
|
||||
# Check for dangerous SQL procedures (filter-specific)
|
||||
v_lower = value.lower()
|
||||
if "xp_cmdshell" in v_lower or "sp_executesql" in v_lower:
|
||||
raise ValueError("Filter value contains potentially malicious SQL procedures.")
|
||||
|
||||
# SQL injection patterns specific to filter values
|
||||
sql_patterns = [
|
||||
r";\s*(DROP|DELETE|INSERT|UPDATE|CREATE|ALTER|EXEC|EXECUTE)\b",
|
||||
r"'\s*OR\s*'",
|
||||
r"'\s*AND\s*'",
|
||||
r"--\s*",
|
||||
r"/\*",
|
||||
r"UNION\s+SELECT",
|
||||
]
|
||||
for pattern in sql_patterns:
|
||||
if re.search(pattern, value, re.IGNORECASE):
|
||||
raise ValueError(
|
||||
"Filter value contains potentially malicious SQL patterns."
|
||||
)
|
||||
|
||||
# Check for shell metacharacters that could indicate injection attempts
|
||||
# Note: We allow '&' alone as it's common in text ("A & B") and is only
|
||||
# dangerous in shell contexts, not in database queries
|
||||
if re.search(r"[;|`$()]", value):
|
||||
raise ValueError("Filter value contains potentially unsafe shell characters.")
|
||||
|
||||
# Check for hex encoding
|
||||
if re.search(r"\\x[0-9a-fA-F]{2}", value):
|
||||
raise ValueError("Filter value contains hex encoding which is not allowed.")
|
||||
|
||||
# Remove dangerous Unicode characters
|
||||
value = _remove_dangerous_unicode(value)
|
||||
|
||||
return value
|
||||
@@ -17,65 +17,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional, TypedDict, Union
|
||||
from urllib import request
|
||||
from urllib.error import URLError
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from celery.beat import SchedulingError
|
||||
from celery.utils.log import get_task_logger
|
||||
from flask import current_app
|
||||
from selenium.common.exceptions import WebDriverException
|
||||
from sqlalchemy import and_, func
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from superset import db, security_manager
|
||||
from superset.extensions import celery_app
|
||||
from superset.models.core import Log
|
||||
from superset.models.dashboard import Dashboard
|
||||
from superset.models.slice import Slice
|
||||
from superset.tags.models import Tag, TaggedObject
|
||||
from superset.tasks.exceptions import ExecutorNotFoundError, InvalidExecutorError
|
||||
from superset.tasks.utils import fetch_csrf_token, get_executor
|
||||
from superset.utils import json
|
||||
from superset.utils.date_parser import parse_human_datetime
|
||||
from superset.utils.machine_auth import MachineAuthProvider
|
||||
from superset.utils.urls import get_url_path, is_secure_url
|
||||
from superset.utils.webdriver import WebDriverSelenium
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
|
||||
class CacheWarmupPayload(TypedDict, total=False):
|
||||
chart_id: int
|
||||
dashboard_id: int | None
|
||||
|
||||
|
||||
class CacheWarmupTask(TypedDict):
|
||||
payload: CacheWarmupPayload
|
||||
username: str | None
|
||||
|
||||
|
||||
def get_task(chart: Slice, dashboard: Optional[Dashboard] = None) -> CacheWarmupTask:
|
||||
"""Return task for warming up a given chart/table cache."""
|
||||
executors = current_app.config["CACHE_WARMUP_EXECUTORS"]
|
||||
payload: CacheWarmupPayload = {"chart_id": chart.id}
|
||||
if dashboard:
|
||||
payload["dashboard_id"] = dashboard.id
|
||||
|
||||
username: str | None
|
||||
try:
|
||||
executor = get_executor(executors, chart)
|
||||
username = executor[1]
|
||||
except (ExecutorNotFoundError, InvalidExecutorError):
|
||||
username = None
|
||||
|
||||
return {"payload": payload, "username": username}
|
||||
def get_dash_url(dashboard: Dashboard) -> str:
|
||||
"""Return external URL for warming up a given dashboard cache."""
|
||||
with current_app.test_request_context():
|
||||
baseurl = (
|
||||
# when running this as an async task, drop the request context with
|
||||
# app.test_request_context()
|
||||
current_app.config.get("WEBDRIVER_BASEURL")
|
||||
or "{SUPERSET_WEBSERVER_PROTOCOL}://"
|
||||
"{SUPERSET_WEBSERVER_ADDRESS}:"
|
||||
"{SUPERSET_WEBSERVER_PORT}".format(**current_app.config)
|
||||
)
|
||||
return f"{baseurl}{dashboard.url}"
|
||||
|
||||
|
||||
class Strategy: # pylint: disable=too-few-public-methods
|
||||
"""
|
||||
A cache warm up strategy.
|
||||
|
||||
Each strategy defines a `get_tasks` method that returns a list of tasks to
|
||||
send to the `/api/v1/chart/warm_up_cache` endpoint.
|
||||
Each strategy defines a `get_urls` method that returns a list of dashboard URLs to
|
||||
warm up using WebDriver.
|
||||
|
||||
Strategies can be configured in `superset/config.py`:
|
||||
|
||||
@@ -96,15 +77,16 @@ class Strategy: # pylint: disable=too-few-public-methods
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def get_tasks(self) -> list[CacheWarmupTask]:
|
||||
raise NotImplementedError("Subclasses must implement get_tasks!")
|
||||
def get_urls(self) -> list[str]:
|
||||
raise NotImplementedError("Subclasses must implement get_urls!")
|
||||
|
||||
|
||||
class DummyStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
"""
|
||||
Warm up all charts.
|
||||
Warm up all published dashboards.
|
||||
|
||||
This is a dummy strategy that will fetch all charts. Can be configured by:
|
||||
This is a dummy strategy that will fetch all published dashboards.
|
||||
Can be configured by:
|
||||
|
||||
beat_schedule = {
|
||||
'cache-warmup-hourly': {
|
||||
@@ -118,8 +100,16 @@ class DummyStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
|
||||
name = "dummy"
|
||||
|
||||
def get_tasks(self) -> list[CacheWarmupTask]:
|
||||
return [get_task(chart) for chart in db.session.query(Slice).all()]
|
||||
def get_urls(self) -> list[str]:
|
||||
# Use selectinload to avoid N+1 queries when checking dashboard.slices
|
||||
dashboards = (
|
||||
db.session.query(Dashboard)
|
||||
.options(selectinload(Dashboard.slices))
|
||||
.filter(Dashboard.published.is_(True))
|
||||
.all()
|
||||
)
|
||||
|
||||
return [get_dash_url(dashboard) for dashboard in dashboards if dashboard.slices]
|
||||
|
||||
|
||||
class TopNDashboardsStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
@@ -147,7 +137,7 @@ class TopNDashboardsStrategy(Strategy): # pylint: disable=too-few-public-method
|
||||
self.top_n = top_n
|
||||
self.since = parse_human_datetime(since) if since else None
|
||||
|
||||
def get_tasks(self) -> list[CacheWarmupTask]:
|
||||
def get_urls(self) -> list[str]:
|
||||
records = (
|
||||
db.session.query(Log.dashboard_id, func.count(Log.dashboard_id))
|
||||
.filter(and_(Log.dashboard_id.isnot(None), Log.dttm >= self.since))
|
||||
@@ -161,11 +151,7 @@ class TopNDashboardsStrategy(Strategy): # pylint: disable=too-few-public-method
|
||||
db.session.query(Dashboard).filter(Dashboard.id.in_(dash_ids)).all()
|
||||
)
|
||||
|
||||
return [
|
||||
get_task(chart, dashboard)
|
||||
for dashboard in dashboards
|
||||
for chart in dashboard.slices
|
||||
]
|
||||
return [get_dash_url(dashboard) for dashboard in dashboards]
|
||||
|
||||
|
||||
class DashboardTagsStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
@@ -190,8 +176,8 @@ class DashboardTagsStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
super().__init__()
|
||||
self.tags = tags or []
|
||||
|
||||
def get_tasks(self) -> list[CacheWarmupTask]:
|
||||
tasks = []
|
||||
def get_urls(self) -> list[str]:
|
||||
urls = []
|
||||
tags = db.session.query(Tag).filter(Tag.name.in_(self.tags)).all()
|
||||
tag_ids = [tag.id for tag in tags]
|
||||
|
||||
@@ -211,73 +197,14 @@ class DashboardTagsStrategy(Strategy): # pylint: disable=too-few-public-methods
|
||||
Dashboard.id.in_(dash_ids)
|
||||
)
|
||||
for dashboard in tagged_dashboards:
|
||||
for chart in dashboard.slices:
|
||||
tasks.append(get_task(chart))
|
||||
urls.append(get_dash_url(dashboard))
|
||||
|
||||
# add charts that are tagged
|
||||
tagged_objects = (
|
||||
db.session.query(TaggedObject)
|
||||
.filter(
|
||||
and_(
|
||||
TaggedObject.object_type == "chart",
|
||||
TaggedObject.tag_id.in_(tag_ids),
|
||||
)
|
||||
)
|
||||
.all()
|
||||
)
|
||||
chart_ids = [tagged_object.object_id for tagged_object in tagged_objects]
|
||||
tagged_charts = db.session.query(Slice).filter(Slice.id.in_(chart_ids))
|
||||
for chart in tagged_charts:
|
||||
tasks.append(get_task(chart))
|
||||
|
||||
return tasks
|
||||
return urls
|
||||
|
||||
|
||||
strategies = [DummyStrategy, TopNDashboardsStrategy, DashboardTagsStrategy]
|
||||
|
||||
|
||||
@celery_app.task(name="fetch_url")
|
||||
def fetch_url(data: str, headers: dict[str, str]) -> dict[str, str]:
|
||||
"""
|
||||
Celery job to fetch url
|
||||
"""
|
||||
result = {}
|
||||
try:
|
||||
url = get_url_path("ChartRestApi.warm_up_cache")
|
||||
|
||||
if is_secure_url(url):
|
||||
logger.info("URL '%s' is secure. Adding Referer header.", url)
|
||||
headers.update({"Referer": url})
|
||||
|
||||
# Fetch CSRF token for API request
|
||||
headers.update(fetch_csrf_token(headers))
|
||||
|
||||
logger.info("Fetching %s with payload %s", url, data)
|
||||
req = request.Request( # noqa: S310
|
||||
url, data=bytes(data, "utf-8"), headers=headers, method="PUT"
|
||||
)
|
||||
response = request.urlopen( # pylint: disable=consider-using-with # noqa: S310
|
||||
req, timeout=600
|
||||
)
|
||||
logger.info(
|
||||
"Fetched %s with payload %s, status code: %s", url, data, response.code
|
||||
)
|
||||
if response.code == 200:
|
||||
result = {"success": data, "response": response.read().decode("utf-8")}
|
||||
else:
|
||||
result = {"error": data, "status_code": response.code}
|
||||
logger.error(
|
||||
"Error fetching %s with payload %s, status code: %s",
|
||||
url,
|
||||
data,
|
||||
response.code,
|
||||
)
|
||||
except URLError as err:
|
||||
logger.exception("Error warming up cache!")
|
||||
result = {"error": data, "exception": str(err)}
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(name="cache-warmup")
|
||||
def cache_warmup(
|
||||
strategy_name: str, *args: Any, **kwargs: Any
|
||||
@@ -285,7 +212,7 @@ def cache_warmup(
|
||||
"""
|
||||
Warm up cache.
|
||||
|
||||
This task periodically hits charts to warm up the cache.
|
||||
This task periodically hits dashboards to warm up the cache.
|
||||
|
||||
"""
|
||||
logger.info("Loading strategy")
|
||||
@@ -307,25 +234,33 @@ def cache_warmup(
|
||||
logger.exception(message)
|
||||
return message
|
||||
|
||||
results: dict[str, list[str]] = {"scheduled": [], "errors": []}
|
||||
for task in strategy.get_tasks():
|
||||
username = task["username"]
|
||||
payload = json.dumps(task["payload"])
|
||||
if username:
|
||||
results: dict[str, list[str]] = {"success": [], "errors": []}
|
||||
|
||||
user = security_manager.find_user(
|
||||
username=current_app.config["SUPERSET_CACHE_WARMUP_USER"]
|
||||
)
|
||||
if not user:
|
||||
message = (
|
||||
f"Cache warmup user '{current_app.config['SUPERSET_CACHE_WARMUP_USER']}' "
|
||||
"not found. Please configure SUPERSET_CACHE_WARMUP_USER with a valid "
|
||||
"username."
|
||||
)
|
||||
logger.error(message)
|
||||
return message
|
||||
|
||||
wd = WebDriverSelenium(current_app.config["WEBDRIVER_TYPE"], user=user)
|
||||
|
||||
try:
|
||||
for url in strategy.get_urls():
|
||||
try:
|
||||
user = security_manager.get_user_by_username(username)
|
||||
cookies = MachineAuthProvider.get_auth_cookies(user)
|
||||
headers = {
|
||||
"Cookie": "session=%s" % cookies.get("session", ""),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
logger.info("Scheduling %s", payload)
|
||||
fetch_url.delay(payload, headers)
|
||||
results["scheduled"].append(payload)
|
||||
except SchedulingError:
|
||||
logger.exception("Error scheduling fetch_url for payload: %s", payload)
|
||||
results["errors"].append(payload)
|
||||
else:
|
||||
logger.warning("Executor not found for %s", payload)
|
||||
logger.info("Fetching %s", url)
|
||||
wd.get_screenshot(url, "grid-container")
|
||||
results["success"].append(url)
|
||||
except (WebDriverException, Exception) as ex: # noqa: BLE001
|
||||
logger.exception("Error warming up cache for %s: %s", url, ex)
|
||||
results["errors"].append(url)
|
||||
finally:
|
||||
# Ensure WebDriver is properly cleaned up
|
||||
wd.destroy()
|
||||
|
||||
return results
|
||||
|
||||
@@ -11357,33 +11357,7 @@ msgid "Tree layout"
|
||||
msgstr "Diseño del árbol"
|
||||
|
||||
msgid "Tree orientation"
|
||||
Findings (brief):
|
||||
|
||||
- No git merge conflict markers found (no <<<<<<< / ======= / >>>>>>>).
|
||||
- PO header mismatch: "Language: en" — this is an Spanish file; set to "es".
|
||||
- Duplicate msgid entries with conflicting/empty translations:
|
||||
- " at line %(line)d" — one entry has " en la línea %(line)d", another has an empty msgstr.
|
||||
- "Dashboard cannot be copied due to invalid parameters." — appears multiple times with different/empty msgstr values.
|
||||
- "%(subtitle)s\nThis may be triggered by:\n %(issue)s" — msgstr is empty in one occurrence.
|
||||
- There are other repeated msgids with one occurrence left untranslated (examples: search for repeated msgid strings with one msgstr == "").
|
||||
- Empty translations (examples):
|
||||
- msgid "%(subtitle)s\nThis may be triggered by:\n %(issue)s" → msgstr "".
|
||||
- Several other msgid entries have msgstr "" (scan for msgstr "" occurrences).
|
||||
- Fuzzy entries present (e.g. entries annotated "#, fuzzy") — these need review and removal of the fuzzy flag after correction.
|
||||
- Typo in a translation: msgid "This action will permanently delete the user." → msgstr contains "uduario." (should be "usuario.").
|
||||
|
||||
Recommended next steps:
|
||||
- Fix header Language to "es".
|
||||
- Remove/fix duplicate msgids: consolidate into a single entry and keep the correct translation.
|
||||
- Fill in missing msgstr values (or mark as untranslated intentionally).
|
||||
- Review and resolve fuzzy entries, then remove the "fuzzy" flag.
|
||||
- Fix obvious typos (e.g., "uduario" → "usuario").
|
||||
|
||||
If you want, I can produce a patch that:
|
||||
- updates header Language to "es",
|
||||
- removes duplicate entries by keeping the first translated occurrence,
|
||||
- lists all msgids with empty msgstr for you to translate,
|
||||
or show exact locations (line ranges) for each problem. Which would you prefer?
|
||||
msgstr "Orientación del árbol"
|
||||
|
||||
msgid "Treemap"
|
||||
msgstr "Diagrama de árbol"
|
||||
|
||||
@@ -33,8 +33,8 @@ from superset.utils.urls import modify_url_query
|
||||
from superset.utils.webdriver import (
|
||||
ChartStandaloneMode,
|
||||
DashboardStandaloneMode,
|
||||
WebDriver,
|
||||
WebDriverPlaywright,
|
||||
WebDriverProxy,
|
||||
WebDriverSelenium,
|
||||
WindowSize,
|
||||
)
|
||||
@@ -188,7 +188,9 @@ class BaseScreenshot:
|
||||
self.url = url
|
||||
self.screenshot = None
|
||||
|
||||
def driver(self, window_size: WindowSize | None = None) -> WebDriver:
|
||||
def driver(
|
||||
self, window_size: WindowSize | None = None, user: User | None = None
|
||||
) -> WebDriverProxy:
|
||||
window_size = window_size or self.window_size
|
||||
if feature_flag_manager.is_feature_enabled("PLAYWRIGHT_REPORTS_AND_THUMBNAILS"):
|
||||
# Try to use Playwright if available (supports WebGL/DeckGL, unlike Cypress)
|
||||
@@ -204,13 +206,13 @@ class BaseScreenshot:
|
||||
)
|
||||
|
||||
# Use Selenium as default/fallback
|
||||
return WebDriverSelenium(self.driver_type, window_size)
|
||||
return WebDriverSelenium(self.driver_type, window_size, user)
|
||||
|
||||
def get_screenshot(
|
||||
self, user: User, window_size: WindowSize | None = None
|
||||
) -> bytes | None:
|
||||
driver = self.driver(window_size)
|
||||
self.screenshot = driver.get_screenshot(self.url, self.element, user)
|
||||
driver = self.driver(window_size, user)
|
||||
self.screenshot = driver.get_screenshot(self.url, self.element)
|
||||
return self.screenshot
|
||||
|
||||
def get_cache_key(
|
||||
|
||||
@@ -159,7 +159,9 @@ class WebDriverProxy(ABC):
|
||||
self._screenshot_load_wait = app.config["SCREENSHOT_LOAD_WAIT"]
|
||||
|
||||
@abstractmethod
|
||||
def get_screenshot(self, url: str, element_name: str, user: User) -> bytes | None:
|
||||
def get_screenshot(
|
||||
self, url: str, element_name: str, user: User | None = None
|
||||
) -> bytes | None:
|
||||
"""
|
||||
Run webdriver and return a screenshot
|
||||
"""
|
||||
@@ -224,7 +226,7 @@ class WebDriverPlaywright(WebDriverProxy):
|
||||
return element.screenshot()
|
||||
|
||||
def get_screenshot( # pylint: disable=too-many-locals, too-many-statements # noqa: C901
|
||||
self, url: str, element_name: str, user: User
|
||||
self, url: str, element_name: str, user: User | None = None
|
||||
) -> bytes | None:
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
logger.info(
|
||||
@@ -252,7 +254,8 @@ class WebDriverPlaywright(WebDriverProxy):
|
||||
context.set_default_timeout(
|
||||
app.config["SCREENSHOT_PLAYWRIGHT_DEFAULT_TIMEOUT"]
|
||||
)
|
||||
self.auth(user, context)
|
||||
if user:
|
||||
self.auth(user, context)
|
||||
page = context.new_page()
|
||||
try:
|
||||
page.goto(
|
||||
@@ -318,7 +321,7 @@ class WebDriverPlaywright(WebDriverProxy):
|
||||
logger.debug(
|
||||
"Taking a PNG screenshot of url %s as user %s",
|
||||
url,
|
||||
user.username,
|
||||
user.username if user else "None",
|
||||
)
|
||||
if app.config["SCREENSHOT_REPLACE_UNEXPECTED_ERRORS"]:
|
||||
unexpected_errors = WebDriverPlaywright.find_unexpected_errors(page)
|
||||
@@ -399,6 +402,29 @@ class WebDriverPlaywright(WebDriverProxy):
|
||||
|
||||
|
||||
class WebDriverSelenium(WebDriverProxy):
|
||||
def __init__(
|
||||
self,
|
||||
driver_type: str,
|
||||
window: WindowSize | None = None,
|
||||
user: User | None = None,
|
||||
):
|
||||
super().__init__(driver_type, window)
|
||||
self._user = user
|
||||
self._driver: WebDriver | None = None
|
||||
|
||||
def __del__(self) -> None:
|
||||
self._destroy()
|
||||
|
||||
@property
|
||||
def driver(self) -> WebDriver:
|
||||
if not self._driver:
|
||||
self._driver = self._create()
|
||||
assert self._driver # for mypy
|
||||
self._driver.set_window_size(*self._window)
|
||||
if self._user:
|
||||
self._auth(self._user)
|
||||
return self._driver
|
||||
|
||||
def _create_firefox_driver(
|
||||
self, pixel_density: float
|
||||
) -> tuple[type[WebDriver], type[Service], dict[str, Any]]:
|
||||
@@ -456,6 +482,22 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
return config
|
||||
|
||||
def create(self) -> WebDriver:
|
||||
"""Create and return the WebDriver instance.
|
||||
|
||||
This is the public interface for creating the driver. It wraps
|
||||
the internal _create method for backward compatibility.
|
||||
"""
|
||||
return self._create()
|
||||
|
||||
def destroy(self) -> None:
|
||||
"""Destroy the WebDriver instance.
|
||||
|
||||
This is the public interface for cleanup. It wraps the internal
|
||||
_destroy method and should be called when done with the driver.
|
||||
"""
|
||||
self._destroy()
|
||||
|
||||
def _create(self) -> WebDriver:
|
||||
pixel_density = app.config["WEBDRIVER_WINDOW"].get("pixel_density", 1)
|
||||
|
||||
# Get driver class and initial kwargs based on driver type
|
||||
@@ -516,26 +558,32 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
logger.debug("Init selenium driver")
|
||||
return driver_class(**kwargs)
|
||||
|
||||
def auth(self, user: User) -> WebDriver:
|
||||
driver = self.create()
|
||||
def _auth(self, user: User) -> WebDriver:
|
||||
return machine_auth_provider_factory.instance.authenticate_webdriver(
|
||||
driver, user
|
||||
self.driver, user
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def destroy(driver: WebDriver, tries: int = 2) -> None:
|
||||
def _destroy(self) -> None:
|
||||
"""Destroy a driver"""
|
||||
if not self._driver:
|
||||
return
|
||||
|
||||
# This is some very flaky code in selenium. Hence the retries
|
||||
# and catch-all exceptions
|
||||
try:
|
||||
retry_call(driver.close, max_tries=tries)
|
||||
retry_call(
|
||||
self._driver.close,
|
||||
max_tries=app.config["SCREENSHOT_SELENIUM_RETRIES"],
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except # noqa: S110
|
||||
pass
|
||||
try:
|
||||
driver.quit()
|
||||
self._driver.quit()
|
||||
except Exception: # pylint: disable=broad-except # noqa: S110
|
||||
pass
|
||||
|
||||
self._driver = None
|
||||
|
||||
@staticmethod
|
||||
def find_unexpected_errors(driver: WebDriver) -> list[str]:
|
||||
error_messages = []
|
||||
@@ -592,10 +640,15 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
|
||||
return error_messages
|
||||
|
||||
def get_screenshot(self, url: str, element_name: str, user: User) -> bytes | None: # noqa: C901
|
||||
driver = self.auth(user)
|
||||
driver.set_window_size(*self._window)
|
||||
driver.get(url)
|
||||
def get_screenshot( # noqa: C901
|
||||
self, url: str, element_name: str, user: User | None = None
|
||||
) -> bytes | None:
|
||||
# Re-authenticate if a different user is passed
|
||||
if user and user != self._user:
|
||||
self._user = user
|
||||
if self._driver:
|
||||
self._auth(user)
|
||||
self.driver.get(url)
|
||||
img: bytes | None = None
|
||||
selenium_headstart = app.config["SCREENSHOT_SELENIUM_HEADSTART"]
|
||||
logger.debug("Sleeping for %i seconds", selenium_headstart)
|
||||
@@ -607,9 +660,9 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
logger.debug(
|
||||
"Wait for the presence of %s at url: %s", element_name, url
|
||||
)
|
||||
element = WebDriverWait(driver, self._screenshot_locate_wait).until(
|
||||
EC.presence_of_element_located((By.CLASS_NAME, element_name))
|
||||
)
|
||||
element = WebDriverWait(
|
||||
self.driver, self._screenshot_locate_wait
|
||||
).until(EC.presence_of_element_located((By.CLASS_NAME, element_name)))
|
||||
except TimeoutException:
|
||||
logger.exception("Selenium timed out requesting url %s", url)
|
||||
raise
|
||||
@@ -617,7 +670,7 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
try:
|
||||
# chart containers didn't render
|
||||
logger.debug("Wait for chart containers to draw at url: %s", url)
|
||||
WebDriverWait(driver, self._screenshot_locate_wait).until(
|
||||
WebDriverWait(self.driver, self._screenshot_locate_wait).until(
|
||||
EC.visibility_of_all_elements_located(
|
||||
(By.CLASS_NAME, "chart-container")
|
||||
)
|
||||
@@ -626,7 +679,7 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
logger.info("Timeout Exception caught")
|
||||
# Fallback to allow a screenshot of an empty dashboard
|
||||
try:
|
||||
WebDriverWait(driver, 0).until(
|
||||
WebDriverWait(self.driver, 0).until(
|
||||
EC.visibility_of_all_elements_located(
|
||||
(By.CLASS_NAME, "grid-container")
|
||||
)
|
||||
@@ -643,7 +696,7 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
logger.debug(
|
||||
"Wait for loading element of charts to be gone at url: %s", url
|
||||
)
|
||||
WebDriverWait(driver, self._screenshot_load_wait).until_not(
|
||||
WebDriverWait(self.driver, self._screenshot_load_wait).until_not(
|
||||
EC.presence_of_all_elements_located((By.CLASS_NAME, "loading"))
|
||||
)
|
||||
except TimeoutException:
|
||||
@@ -658,11 +711,13 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
logger.debug(
|
||||
"Taking a PNG screenshot of url %s as user %s",
|
||||
url,
|
||||
user.username,
|
||||
user.username if user else "None",
|
||||
)
|
||||
|
||||
if app.config["SCREENSHOT_REPLACE_UNEXPECTED_ERRORS"]:
|
||||
unexpected_errors = WebDriverSelenium.find_unexpected_errors(driver)
|
||||
unexpected_errors = WebDriverSelenium.find_unexpected_errors(
|
||||
self.driver
|
||||
)
|
||||
if unexpected_errors:
|
||||
logger.warning(
|
||||
"%i errors found in the screenshot. URL: %s. Errors are: %s",
|
||||
@@ -689,6 +744,4 @@ class WebDriverSelenium(WebDriverProxy):
|
||||
"Encountered an unexpected error when requesting url %s", url
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
self.destroy(driver, app.config["SCREENSHOT_SELENIUM_RETRIES"])
|
||||
return img
|
||||
|
||||
@@ -39,7 +39,7 @@ from superset.tasks.cache import (
|
||||
DashboardTagsStrategy,
|
||||
TopNDashboardsStrategy,
|
||||
)
|
||||
from superset.utils.urls import get_url_host # noqa: F401
|
||||
from superset.utils.urls import get_url_host
|
||||
|
||||
from tests.integration_tests.base_tests import SupersetTestCase
|
||||
from tests.integration_tests.constants import ADMIN_USERNAME
|
||||
@@ -82,15 +82,9 @@ class TestCacheWarmUp(SupersetTestCase):
|
||||
self.client.get(f"/superset/dashboard/{dash.id}/")
|
||||
|
||||
strategy = TopNDashboardsStrategy(1)
|
||||
result = strategy.get_tasks()
|
||||
expected = [
|
||||
{
|
||||
"payload": {"chart_id": chart.id, "dashboard_id": dash.id},
|
||||
"username": "admin",
|
||||
}
|
||||
for chart in dash.slices
|
||||
]
|
||||
assert len(result) == len(expected)
|
||||
result = sorted(strategy.get_urls())
|
||||
expected = sorted([f"{get_url_host()}{dash.url}"])
|
||||
assert result == expected
|
||||
|
||||
def reset_tag(self, tag):
|
||||
"""Remove associated object from tag, used to reset tests"""
|
||||
@@ -108,39 +102,27 @@ class TestCacheWarmUp(SupersetTestCase):
|
||||
self.reset_tag(tag1)
|
||||
|
||||
strategy = DashboardTagsStrategy(["tag1"])
|
||||
assert strategy.get_tasks() == []
|
||||
result = sorted(strategy.get_urls())
|
||||
expected = []
|
||||
assert result == expected
|
||||
|
||||
# tag dashboard 'births' with `tag1`
|
||||
tag1 = get_tag("tag1", db.session, TagType.custom)
|
||||
dash = self.get_dash_by_slug("births")
|
||||
tag1_payloads = [{"chart_id": chart.id} for chart in dash.slices]
|
||||
tag1_urls = [f"{get_url_host()}{dash.url}"]
|
||||
tagged_object = TaggedObject(
|
||||
tag_id=tag1.id, object_id=dash.id, object_type=ObjectType.dashboard
|
||||
)
|
||||
db.session.add(tagged_object)
|
||||
db.session.commit()
|
||||
|
||||
assert len(strategy.get_tasks()) == len(tag1_payloads)
|
||||
result = sorted(strategy.get_urls())
|
||||
assert result == tag1_urls
|
||||
|
||||
strategy = DashboardTagsStrategy(["tag2"])
|
||||
tag2 = get_tag("tag2", db.session, TagType.custom)
|
||||
self.reset_tag(tag2)
|
||||
|
||||
assert strategy.get_tasks() == []
|
||||
|
||||
# tag first slice
|
||||
dash = self.get_dash_by_slug("unicode-test")
|
||||
chart = dash.slices[0]
|
||||
tag2_payloads = [{"chart_id": chart.id}]
|
||||
object_id = chart.id
|
||||
tagged_object = TaggedObject(
|
||||
tag_id=tag2.id, object_id=object_id, object_type=ObjectType.chart
|
||||
)
|
||||
db.session.add(tagged_object)
|
||||
db.session.commit()
|
||||
|
||||
assert len(strategy.get_tasks()) == len(tag2_payloads)
|
||||
|
||||
strategy = DashboardTagsStrategy(["tag1", "tag2"])
|
||||
|
||||
assert len(strategy.get_tasks()) == len(tag1_payloads + tag2_payloads)
|
||||
result = sorted(strategy.get_urls())
|
||||
expected = []
|
||||
assert result == expected
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration_tests.test_app import app
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url, expected_referer",
|
||||
[
|
||||
("http://base-url", None),
|
||||
("http://base-url/", None),
|
||||
("https://base-url", "https://base-url/api/v1/chart/warm_up_cache"),
|
||||
("https://base-url/", "https://base-url/api/v1/chart/warm_up_cache"),
|
||||
],
|
||||
ids=[
|
||||
"Without trailing slash (HTTP)",
|
||||
"With trailing slash (HTTP)",
|
||||
"Without trailing slash (HTTPS)",
|
||||
"With trailing slash (HTTPS)",
|
||||
],
|
||||
)
|
||||
@mock.patch("superset.tasks.cache.fetch_csrf_token")
|
||||
@mock.patch("superset.tasks.cache.request.Request")
|
||||
@mock.patch("superset.tasks.cache.request.urlopen")
|
||||
@mock.patch("superset.tasks.cache.is_secure_url")
|
||||
def test_fetch_url(
|
||||
mock_is_secure_url,
|
||||
mock_urlopen,
|
||||
mock_request_cls,
|
||||
mock_fetch_csrf_token,
|
||||
base_url,
|
||||
expected_referer,
|
||||
):
|
||||
from superset.tasks.cache import fetch_url
|
||||
|
||||
mock_request = mock.MagicMock()
|
||||
mock_request_cls.return_value = mock_request
|
||||
|
||||
mock_urlopen.return_value = mock.MagicMock()
|
||||
mock_urlopen.return_value.code = 200
|
||||
|
||||
# Mock the URL validation to return True for HTTPS and False for HTTP
|
||||
mock_is_secure_url.return_value = base_url.startswith("https")
|
||||
|
||||
initial_headers = {"Cookie": "cookie", "key": "value"}
|
||||
csrf_headers = initial_headers | {"X-CSRF-Token": "csrf_token"}
|
||||
|
||||
# Conditionally add the Referer header and assert its presence
|
||||
if expected_referer:
|
||||
csrf_headers = csrf_headers | {"Referer": expected_referer}
|
||||
assert csrf_headers["Referer"] == expected_referer
|
||||
|
||||
mock_fetch_csrf_token.return_value = csrf_headers
|
||||
|
||||
app.config["WEBDRIVER_BASEURL"] = base_url
|
||||
data = "data"
|
||||
data_encoded = b"data"
|
||||
|
||||
result = fetch_url(data, initial_headers)
|
||||
|
||||
expected_url = (
|
||||
f"{base_url}/api/v1/chart/warm_up_cache"
|
||||
if not base_url.endswith("/")
|
||||
else f"{base_url}api/v1/chart/warm_up_cache"
|
||||
)
|
||||
|
||||
mock_fetch_csrf_token.assert_called_once_with(initial_headers)
|
||||
|
||||
mock_request_cls.assert_called_once_with(
|
||||
expected_url, # Use the dynamic URL based on base_url
|
||||
data=data_encoded,
|
||||
headers=csrf_headers,
|
||||
method="PUT",
|
||||
)
|
||||
# assert the same Request object is used
|
||||
mock_urlopen.assert_called_once_with(mock_request, timeout=mock.ANY)
|
||||
|
||||
assert data == result["success"]
|
||||
@@ -1,77 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration_tests.test_app import app
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"http://base-url",
|
||||
"http://base-url/",
|
||||
"https://base-url",
|
||||
"https://base-url/",
|
||||
],
|
||||
ids=[
|
||||
"Without trailing slash (HTTP)",
|
||||
"With trailing slash (HTTP)",
|
||||
"Without trailing slash (HTTPS)",
|
||||
"With trailing slash (HTTPS)",
|
||||
],
|
||||
)
|
||||
@mock.patch("superset.tasks.cache.request.Request")
|
||||
@mock.patch("superset.tasks.cache.request.urlopen")
|
||||
def test_fetch_csrf_token(mock_urlopen, mock_request_cls, base_url, app_context):
|
||||
from superset.tasks.utils import fetch_csrf_token
|
||||
|
||||
mock_request = mock.MagicMock()
|
||||
mock_request_cls.return_value = mock_request
|
||||
|
||||
mock_response = mock.MagicMock()
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_response
|
||||
|
||||
mock_response.status = 200
|
||||
mock_response.read.return_value = b'{"result": "csrf_token"}'
|
||||
mock_response.headers.get_all.return_value = [
|
||||
"session=new_session_cookie",
|
||||
"async-token=websocket_cookie",
|
||||
]
|
||||
|
||||
app.config["WEBDRIVER_BASEURL"] = base_url
|
||||
headers = {"Cookie": "original_session_cookie"}
|
||||
|
||||
result_headers = fetch_csrf_token(headers)
|
||||
|
||||
expected_url = (
|
||||
f"{base_url}/api/v1/security/csrf_token/"
|
||||
if not base_url.endswith("/")
|
||||
else f"{base_url}api/v1/security/csrf_token/"
|
||||
)
|
||||
|
||||
mock_request_cls.assert_called_with(
|
||||
expected_url,
|
||||
headers=headers,
|
||||
method="GET",
|
||||
)
|
||||
|
||||
assert result_headers["X-CSRF-Token"] == "csrf_token"
|
||||
assert result_headers["Cookie"] == "session=new_session_cookie" # Updated assertion
|
||||
# assert the same Request object is used
|
||||
mock_urlopen.assert_called_once_with(mock_request, timeout=mock.ANY)
|
||||
@@ -147,31 +147,31 @@ class TestWebDriverSelenium(SupersetTestCase):
|
||||
def test_screenshot_selenium_headstart(
|
||||
self, mock_sleep, mock_webdriver, mock_webdriver_wait
|
||||
):
|
||||
webdriver = WebDriverSelenium("firefox")
|
||||
user = security_manager.get_user_by_username(ADMIN_USERNAME)
|
||||
webdriver = WebDriverSelenium("firefox", user=user)
|
||||
url = get_url_path("Superset.slice", slice_id=1, standalone="true")
|
||||
app.config["SCREENSHOT_SELENIUM_HEADSTART"] = 5
|
||||
webdriver.get_screenshot(url, "chart-container", user=user)
|
||||
webdriver.get_screenshot(url, "chart-container")
|
||||
assert mock_sleep.call_args_list[0] == call(5)
|
||||
|
||||
@patch("superset.utils.webdriver.WebDriverWait")
|
||||
@patch("superset.utils.webdriver.firefox")
|
||||
def test_screenshot_selenium_locate_wait(self, mock_webdriver, mock_webdriver_wait):
|
||||
app.config["SCREENSHOT_LOCATE_WAIT"] = 15
|
||||
webdriver = WebDriverSelenium("firefox")
|
||||
user = security_manager.get_user_by_username(ADMIN_USERNAME)
|
||||
webdriver = WebDriverSelenium("firefox", user=user)
|
||||
url = get_url_path("Superset.slice", slice_id=1, standalone="true")
|
||||
webdriver.get_screenshot(url, "chart-container", user=user)
|
||||
webdriver.get_screenshot(url, "chart-container")
|
||||
assert mock_webdriver_wait.call_args_list[0] == call(ANY, 15)
|
||||
|
||||
@patch("superset.utils.webdriver.WebDriverWait")
|
||||
@patch("superset.utils.webdriver.firefox")
|
||||
def test_screenshot_selenium_load_wait(self, mock_webdriver, mock_webdriver_wait):
|
||||
app.config["SCREENSHOT_LOAD_WAIT"] = 15
|
||||
webdriver = WebDriverSelenium("firefox")
|
||||
user = security_manager.get_user_by_username(ADMIN_USERNAME)
|
||||
webdriver = WebDriverSelenium("firefox", user=user)
|
||||
url = get_url_path("Superset.slice", slice_id=1, standalone="true")
|
||||
webdriver.get_screenshot(url, "chart-container", user=user)
|
||||
webdriver.get_screenshot(url, "chart-container")
|
||||
assert mock_webdriver_wait.call_args_list[2] == call(ANY, 15)
|
||||
|
||||
@patch("superset.utils.webdriver.WebDriverWait")
|
||||
@@ -180,11 +180,11 @@ class TestWebDriverSelenium(SupersetTestCase):
|
||||
def test_screenshot_selenium_animation_wait(
|
||||
self, mock_sleep, mock_webdriver, mock_webdriver_wait
|
||||
):
|
||||
webdriver = WebDriverSelenium("firefox")
|
||||
user = security_manager.get_user_by_username(ADMIN_USERNAME)
|
||||
webdriver = WebDriverSelenium("firefox", user=user)
|
||||
url = get_url_path("Superset.slice", slice_id=1, standalone="true")
|
||||
app.config["SCREENSHOT_SELENIUM_ANIMATION_WAIT"] = 4
|
||||
webdriver.get_screenshot(url, "chart-container", user=user)
|
||||
webdriver.get_screenshot(url, "chart-container")
|
||||
assert mock_sleep.call_args_list[1] == call(4)
|
||||
|
||||
|
||||
|
||||
480
tests/unit_tests/mcp_service/utils/test_sanitization.py
Normal file
480
tests/unit_tests/mcp_service/utils/test_sanitization.py
Normal file
@@ -0,0 +1,480 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.mcp_service.utils.sanitization import (
|
||||
_check_dangerous_patterns,
|
||||
_check_sql_patterns,
|
||||
_remove_dangerous_unicode,
|
||||
_strip_html_tags,
|
||||
sanitize_filter_value,
|
||||
sanitize_user_input,
|
||||
)
|
||||
|
||||
# --- _strip_html_tags tests ---
|
||||
|
||||
|
||||
def test_strip_html_tags_plain_text():
|
||||
assert _strip_html_tags("hello world") == "hello world"
|
||||
|
||||
|
||||
def test_strip_html_tags_preserves_ampersand():
|
||||
assert _strip_html_tags("A & B") == "A & B"
|
||||
|
||||
|
||||
def test_strip_html_tags_preserves_multiple_ampersands():
|
||||
assert _strip_html_tags("A & B & C") == "A & B & C"
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_bold_tags():
|
||||
assert _strip_html_tags("<b>hello</b>") == "hello"
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_script_tags():
|
||||
result = _strip_html_tags("<script>alert(1)</script>")
|
||||
assert "<script>" not in result
|
||||
assert "</script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_entity_encoded_script():
|
||||
"""Entity-encoded tags must be decoded and stripped, not passed through."""
|
||||
result = _strip_html_tags("<script>alert(1)</script>")
|
||||
assert "<script>" not in result
|
||||
assert "<script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_double_encoded_script():
|
||||
"""Double-encoded entities must also be decoded and stripped."""
|
||||
result = _strip_html_tags("&lt;script&gt;alert(1)&lt;/script&gt;")
|
||||
assert "<script>" not in result
|
||||
assert "<script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_img_onerror():
|
||||
result = _strip_html_tags('<img src=x onerror="alert(1)">')
|
||||
assert "<img" not in result
|
||||
assert "onerror" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_strips_div_tags():
|
||||
assert _strip_html_tags("<div>content</div>") == "content"
|
||||
|
||||
|
||||
def test_strip_html_tags_preserves_less_than_in_text():
|
||||
"""A bare < not forming a tag should be preserved."""
|
||||
result = _strip_html_tags("5 < 10")
|
||||
assert "5" in result
|
||||
assert "10" in result
|
||||
|
||||
|
||||
def test_strip_html_tags_empty_string():
|
||||
assert _strip_html_tags("") == ""
|
||||
|
||||
|
||||
def test_strip_html_tags_triple_encoded_script():
|
||||
"""Triple-encoded entities must also be decoded and stripped."""
|
||||
result = _strip_html_tags(
|
||||
"&amp;lt;script&amp;gt;alert(1)&amp;lt;/script&amp;gt;"
|
||||
)
|
||||
assert "<script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_mixed_encoded_and_raw():
|
||||
"""Both raw and entity-encoded tags should be stripped."""
|
||||
result = _strip_html_tags("<b>bold</b> and <i>italic</i>")
|
||||
assert "<b>" not in result
|
||||
assert "<i>" not in result
|
||||
assert "bold" in result
|
||||
assert "italic" in result
|
||||
|
||||
|
||||
def test_strip_html_tags_deep_encoding_terminates():
|
||||
"""Verify the iterative decode loop terminates on many encoding layers."""
|
||||
value = "<script>alert(1)</script>"
|
||||
for _ in range(10):
|
||||
value = value.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
result = _strip_html_tags(value)
|
||||
assert "<script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_entity_ampersand():
|
||||
"""& in input should become & in output."""
|
||||
assert _strip_html_tags("A & B") == "A & B"
|
||||
|
||||
|
||||
# --- _check_dangerous_patterns tests ---
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_safe_input():
|
||||
_check_dangerous_patterns("hello world", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_javascript_scheme():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
_check_dangerous_patterns("javascript:alert(1)", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_vbscript_scheme():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
_check_dangerous_patterns("vbscript:MsgBox", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_data_scheme():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
_check_dangerous_patterns("data:text/html,<script>", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_case_insensitive():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
_check_dangerous_patterns("JAVASCRIPT:alert(1)", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_onclick():
|
||||
with pytest.raises(ValueError, match="malicious event handler"):
|
||||
_check_dangerous_patterns("onclick=alert(1)", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_onerror():
|
||||
with pytest.raises(ValueError, match="malicious event handler"):
|
||||
_check_dangerous_patterns("onerror = alert(1)", "test")
|
||||
|
||||
|
||||
def test_check_dangerous_patterns_onload():
|
||||
with pytest.raises(ValueError, match="malicious event handler"):
|
||||
_check_dangerous_patterns("onload=fetch('x')", "test")
|
||||
|
||||
|
||||
# --- _check_sql_patterns tests ---
|
||||
|
||||
|
||||
def test_check_sql_patterns_safe_input():
|
||||
_check_sql_patterns("revenue_total", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_drop_table():
|
||||
with pytest.raises(ValueError, match="unsafe SQL keywords"):
|
||||
_check_sql_patterns("DROP TABLE users", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_delete():
|
||||
with pytest.raises(ValueError, match="unsafe SQL keywords"):
|
||||
_check_sql_patterns("DELETE FROM users", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_semicolon():
|
||||
with pytest.raises(ValueError, match="unsafe characters"):
|
||||
_check_sql_patterns("value; other", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_sql_comment_dash():
|
||||
with pytest.raises(ValueError, match="unsafe characters"):
|
||||
_check_sql_patterns("value -- comment", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_sql_comment_block():
|
||||
with pytest.raises(ValueError, match="unsafe SQL comment"):
|
||||
_check_sql_patterns("value /* comment */", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_pipe():
|
||||
with pytest.raises(ValueError, match="unsafe characters"):
|
||||
_check_sql_patterns("value | other", "test")
|
||||
|
||||
|
||||
def test_check_sql_patterns_case_insensitive():
|
||||
with pytest.raises(ValueError, match="unsafe SQL keywords"):
|
||||
_check_sql_patterns("drop table users", "test")
|
||||
|
||||
|
||||
# --- _remove_dangerous_unicode tests ---
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_plain_text():
|
||||
assert _remove_dangerous_unicode("hello") == "hello"
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_zero_width_space():
|
||||
assert _remove_dangerous_unicode("he\u200bllo") == "hello"
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_zero_width_joiner():
|
||||
assert _remove_dangerous_unicode("he\u200dllo") == "hello"
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_bom():
|
||||
assert _remove_dangerous_unicode("\ufeffhello") == "hello"
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_null_byte():
|
||||
assert _remove_dangerous_unicode("he\x00llo") == "hello"
|
||||
|
||||
|
||||
def test_remove_dangerous_unicode_preserves_normal_unicode():
|
||||
assert _remove_dangerous_unicode("café résumé") == "café résumé"
|
||||
|
||||
|
||||
# --- sanitize_user_input tests ---
|
||||
|
||||
|
||||
def test_sanitize_user_input_plain_text():
|
||||
assert sanitize_user_input("hello", "test") == "hello"
|
||||
|
||||
|
||||
def test_sanitize_user_input_preserves_ampersand():
|
||||
assert sanitize_user_input("A & B", "test") == "A & B"
|
||||
|
||||
|
||||
def test_sanitize_user_input_strips_html():
|
||||
assert sanitize_user_input("<b>hello</b>", "test") == "hello"
|
||||
|
||||
|
||||
def test_sanitize_user_input_none_not_allowed():
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
sanitize_user_input(None, "test")
|
||||
|
||||
|
||||
def test_sanitize_user_input_none_allowed():
|
||||
assert sanitize_user_input(None, "test", allow_empty=True) is None
|
||||
|
||||
|
||||
def test_sanitize_user_input_empty_string_not_allowed():
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
sanitize_user_input("", "test")
|
||||
|
||||
|
||||
def test_sanitize_user_input_empty_string_allowed():
|
||||
assert sanitize_user_input("", "test", allow_empty=True) is None
|
||||
|
||||
|
||||
def test_sanitize_user_input_whitespace_only():
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
sanitize_user_input(" ", "test")
|
||||
|
||||
|
||||
def test_sanitize_user_input_strips_whitespace():
|
||||
assert sanitize_user_input(" hello ", "test") == "hello"
|
||||
|
||||
|
||||
def test_sanitize_user_input_too_long():
|
||||
with pytest.raises(ValueError, match="too long"):
|
||||
sanitize_user_input("a" * 256, "test", max_length=255)
|
||||
|
||||
|
||||
def test_sanitize_user_input_max_length_ok():
|
||||
result = sanitize_user_input("a" * 255, "test", max_length=255)
|
||||
assert result == "a" * 255
|
||||
|
||||
|
||||
def test_sanitize_user_input_blocks_javascript():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
sanitize_user_input("javascript:alert(1)", "test")
|
||||
|
||||
|
||||
def test_sanitize_user_input_blocks_event_handler():
|
||||
with pytest.raises(ValueError, match="malicious event handler"):
|
||||
sanitize_user_input("onclick=alert(1)", "test")
|
||||
|
||||
|
||||
def test_sanitize_user_input_sql_keywords_not_checked_by_default():
|
||||
result = sanitize_user_input("DROP TABLE", "test")
|
||||
assert result == "DROP TABLE"
|
||||
|
||||
|
||||
def test_sanitize_user_input_sql_keywords_checked_when_enabled():
|
||||
with pytest.raises(ValueError, match="unsafe SQL keywords"):
|
||||
sanitize_user_input("DROP TABLE users", "test", check_sql_keywords=True)
|
||||
|
||||
|
||||
def test_sanitize_user_input_removes_zero_width_chars():
|
||||
result = sanitize_user_input("hel\u200blo", "test")
|
||||
assert result == "hello"
|
||||
|
||||
|
||||
def test_sanitize_user_input_xss_entity_encoded():
|
||||
"""Entity-encoded XSS attempts must be neutralized."""
|
||||
result = sanitize_user_input("<script>alert(1)</script>", "test")
|
||||
assert "<script>" not in result
|
||||
|
||||
|
||||
def test_sanitize_user_input_entity_encoded_javascript():
|
||||
"""Entity-encoded javascript: scheme should be caught after decoding."""
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
sanitize_user_input("javascript:alert(1)", "test")
|
||||
|
||||
|
||||
# --- sanitize_filter_value tests ---
|
||||
|
||||
|
||||
def test_sanitize_filter_value_integer():
|
||||
assert sanitize_filter_value(42) == 42
|
||||
|
||||
|
||||
def test_sanitize_filter_value_float():
|
||||
assert sanitize_filter_value(3.14) == 3.14
|
||||
|
||||
|
||||
def test_sanitize_filter_value_bool():
|
||||
assert sanitize_filter_value(True) is True
|
||||
|
||||
|
||||
def test_sanitize_filter_value_plain_string():
|
||||
assert sanitize_filter_value("hello") == "hello"
|
||||
|
||||
|
||||
def test_sanitize_filter_value_preserves_ampersand():
|
||||
assert sanitize_filter_value("A & B") == "A & B"
|
||||
|
||||
|
||||
def test_sanitize_filter_value_strips_html():
|
||||
assert sanitize_filter_value("<b>hello</b>") == "hello"
|
||||
|
||||
|
||||
def test_sanitize_filter_value_too_long():
|
||||
with pytest.raises(ValueError, match="too long"):
|
||||
sanitize_filter_value("a" * 1001)
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_javascript():
|
||||
with pytest.raises(ValueError, match="malicious URL scheme"):
|
||||
sanitize_filter_value("javascript:alert(1)")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_xp_cmdshell():
|
||||
with pytest.raises(ValueError, match="malicious SQL procedures"):
|
||||
sanitize_filter_value("xp_cmdshell('dir')")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_sp_executesql():
|
||||
with pytest.raises(ValueError, match="malicious SQL procedures"):
|
||||
sanitize_filter_value("sp_executesql @stmt")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_union_select():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("' UNION SELECT * FROM users")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_sql_comment():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("value -- drop")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_shell_semicolon():
|
||||
with pytest.raises(ValueError, match="unsafe shell characters"):
|
||||
sanitize_filter_value("value;rm -rf")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_shell_pipe():
|
||||
with pytest.raises(ValueError, match="unsafe shell characters"):
|
||||
sanitize_filter_value("value|cat /etc/passwd")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_backtick():
|
||||
with pytest.raises(ValueError, match="unsafe shell characters"):
|
||||
sanitize_filter_value("`whoami`")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_hex_encoding():
|
||||
with pytest.raises(ValueError, match="hex encoding"):
|
||||
sanitize_filter_value("\\x41\\x42")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_allows_ampersand_alone():
|
||||
"""Ampersand is safe in filter values (only dangerous in shell contexts)."""
|
||||
assert sanitize_filter_value("AT&T") == "AT&T"
|
||||
|
||||
|
||||
def test_sanitize_filter_value_removes_zero_width_chars():
|
||||
result = sanitize_filter_value("hel\u200blo")
|
||||
assert result == "hello"
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_or_injection():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("' OR '1'='1")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_and_injection():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("' AND '1'='1")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_block_comment():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("value /* comment */")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_semicolon_drop():
|
||||
with pytest.raises(ValueError, match="malicious SQL patterns"):
|
||||
sanitize_filter_value("; DROP TABLE users")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_parentheses():
|
||||
with pytest.raises(ValueError, match="unsafe shell characters"):
|
||||
sanitize_filter_value("$(whoami)")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_dollar_sign():
|
||||
with pytest.raises(ValueError, match="unsafe shell characters"):
|
||||
sanitize_filter_value("$HOME")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_blocks_event_handler():
|
||||
with pytest.raises(ValueError, match="malicious event handler"):
|
||||
sanitize_filter_value("onerror=alert(1)")
|
||||
|
||||
|
||||
def test_sanitize_filter_value_xss_entity_encoded():
|
||||
"""Entity-encoded XSS in filter values must be neutralized."""
|
||||
result = sanitize_filter_value("<img src=x onerror=alert(1)>")
|
||||
assert "<img" not in result
|
||||
|
||||
|
||||
# --- Defense-in-depth: verify html.unescape is not used after nh3 ---
|
||||
|
||||
|
||||
def test_strip_html_tags_does_not_unescape_angle_brackets():
|
||||
"""Ensure nh3 entity output is not fully unescaped back to raw HTML.
|
||||
|
||||
nh3.clean may pass through HTML entities (e.g. <script>) from
|
||||
the input without stripping them. A full html.unescape() on nh3's
|
||||
output could reintroduce raw angle brackets, creating an XSS vector.
|
||||
"""
|
||||
# Plain text passes through unchanged
|
||||
result = _strip_html_tags("safe text")
|
||||
assert result == "safe text"
|
||||
|
||||
# Verify ampersand preservation still works
|
||||
result = _strip_html_tags("A & B")
|
||||
assert result == "A & B"
|
||||
|
||||
# Verify real tags are stripped
|
||||
result = _strip_html_tags("<script>alert(1)</script>")
|
||||
assert "<script>" not in result
|
||||
|
||||
# Entity-encoded script tags must not become real tags in the output
|
||||
result = _strip_html_tags("<script>alert(1)</script>")
|
||||
assert "<script>" not in result
|
||||
assert "</script>" not in result
|
||||
|
||||
|
||||
def test_strip_html_tags_img_onerror_entity_bypass():
|
||||
"""Entity-encoded img/onerror should not survive sanitization."""
|
||||
result = _strip_html_tags("<img src=x onerror=alert(1)>")
|
||||
assert "<img" not in result
|
||||
assert "onerror" not in result
|
||||
Reference in New Issue
Block a user