mirror of
https://github.com/apache/superset.git
synced 2026-06-20 23:19:18 +00:00
Compare commits
9 Commits
snowflake-
...
prefer-bin
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5d32c8834d | ||
|
|
40164300e5 | ||
|
|
c444eed63e | ||
|
|
1df5e59fdf | ||
|
|
77f66e7434 | ||
|
|
2c81eb6c39 | ||
|
|
09c4afc894 | ||
|
|
229d92590a | ||
|
|
f4f516c64c |
125
.cursor/rules/dev-standard.mdc
Normal file
125
.cursor/rules/dev-standard.mdc
Normal file
@@ -0,0 +1,125 @@
|
||||
---
|
||||
description: Apache Superset development standards and guidelines for Cursor IDE
|
||||
globs: ["**/*.py", "**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.sql", "**/*.md"]
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Apache Superset Development Standards for Cursor IDE
|
||||
|
||||
Apache Superset is a data visualization platform with Flask/Python backend and React/TypeScript frontend.
|
||||
|
||||
## ⚠️ CRITICAL: Ongoing Refactors (What NOT to Do)
|
||||
|
||||
**These migrations are actively happening - avoid deprecated patterns:**
|
||||
|
||||
### Frontend Modernization
|
||||
- **NO `any` types** - Use proper TypeScript types
|
||||
- **NO JavaScript files** - Convert to TypeScript (.ts/.tsx)
|
||||
- **NO Enzyme** - Use React Testing Library/Jest (Enzyme fully removed)
|
||||
- **Use @superset-ui/core** - Don't import Ant Design directly
|
||||
|
||||
### Testing Strategy Migration
|
||||
- **Prefer unit tests** over integration tests
|
||||
- **Prefer integration tests** over Cypress end-to-end tests
|
||||
- **Cypress is last resort** - Actively moving away from Cypress
|
||||
- **Use Jest + React Testing Library** for component testing
|
||||
|
||||
### Backend Type Safety
|
||||
- **Add type hints** - All new Python code needs proper typing
|
||||
- **MyPy compliance** - Run `pre-commit run mypy` to validate
|
||||
- **SQLAlchemy typing** - Use proper model annotations
|
||||
|
||||
## Code Standards
|
||||
|
||||
### TypeScript Frontend
|
||||
- **NO `any` types** - Use proper TypeScript
|
||||
- **Functional components** with hooks
|
||||
- **@superset-ui/core** for UI components (not direct antd)
|
||||
- **Jest** for testing (NO Enzyme)
|
||||
- **Redux** for global state, hooks for local
|
||||
|
||||
### Python Backend
|
||||
- **Type hints required** for all new code
|
||||
- **MyPy compliant** - run `pre-commit run mypy`
|
||||
- **SQLAlchemy models** with proper typing
|
||||
- **pytest** for testing
|
||||
|
||||
### Apache License Headers
|
||||
- **New files require ASF license headers** - When creating new code files, include the standard Apache Software Foundation license header
|
||||
- **LLM instruction files are excluded** - Files like LLMS.md, CLAUDE.md, etc. are in `.rat-excludes` to avoid header token overhead
|
||||
|
||||
## Key Directory Structure
|
||||
|
||||
```
|
||||
superset/
|
||||
├── superset/ # Python backend (Flask, SQLAlchemy)
|
||||
│ ├── views/api/ # REST API endpoints
|
||||
│ ├── models/ # Database models
|
||||
│ └── connectors/ # Database connections
|
||||
├── superset-frontend/src/ # React TypeScript frontend
|
||||
│ ├── components/ # Reusable components
|
||||
│ ├── explore/ # Chart builder
|
||||
│ ├── dashboard/ # Dashboard interface
|
||||
│ └── SqlLab/ # SQL editor
|
||||
├── superset-frontend/packages/
|
||||
│ └── superset-ui-core/ # UI component library (USE THIS)
|
||||
├── tests/ # Python/integration tests
|
||||
├── docs/ # Documentation (UPDATE FOR CHANGES)
|
||||
└── UPDATING.md # Breaking changes log
|
||||
```
|
||||
|
||||
## Architecture Patterns
|
||||
|
||||
### Dataset-Centric Approach
|
||||
Charts built from enriched datasets containing:
|
||||
- Dimension columns with labels/descriptions
|
||||
- Predefined metrics as SQL expressions
|
||||
- Self-service analytics within defined contexts
|
||||
|
||||
### Security & Features
|
||||
- **RBAC**: Role-based access via Flask-AppBuilder
|
||||
- **Feature flags**: Control feature rollouts
|
||||
- **Row-level security**: SQL-based data access control
|
||||
|
||||
## Test Utilities
|
||||
|
||||
### Python Test Helpers
|
||||
- **`SupersetTestCase`** - Base class in `tests/integration_tests/base_tests.py`
|
||||
- **`@with_config`** - Config mocking decorator
|
||||
- **`@with_feature_flags`** - Feature flag testing
|
||||
- **`login_as()`, `login_as_admin()`** - Authentication helpers
|
||||
- **`create_dashboard()`, `create_slice()`** - Data setup utilities
|
||||
|
||||
### TypeScript Test Helpers
|
||||
- **`superset-frontend/spec/helpers/testing-library.tsx`** - Custom render() with providers
|
||||
- **`createWrapper()`** - Redux/Router/Theme wrapper
|
||||
- **`selectOption()`** - Select component helper
|
||||
- **React Testing Library** - NO Enzyme (removed)
|
||||
|
||||
## Pre-commit Validation
|
||||
|
||||
**Use pre-commit hooks for quality validation:**
|
||||
|
||||
```bash
|
||||
# Install hooks
|
||||
pre-commit install
|
||||
|
||||
# Quick validation (faster than --all-files)
|
||||
pre-commit run # Staged files only
|
||||
pre-commit run mypy # Python type checking
|
||||
pre-commit run prettier # Code formatting
|
||||
pre-commit run eslint # Frontend linting
|
||||
```
|
||||
|
||||
## Development Guidelines
|
||||
|
||||
- **Documentation**: Update docs/ for any user-facing changes
|
||||
- **Breaking Changes**: Add to UPDATING.md
|
||||
- **Docstrings**: Required for new functions/classes
|
||||
- **Follow existing patterns**: Mimic code style, use existing libraries and utilities
|
||||
- **Type Safety**: This codebase is actively modernizing toward full TypeScript and type safety
|
||||
- **Always run `pre-commit run`** to validate changes before committing
|
||||
|
||||
---
|
||||
|
||||
**Note**: This codebase is actively modernizing toward full TypeScript and type safety. Always run `pre-commit run` to validate changes. Follow the ongoing refactors section to avoid deprecated patterns.
|
||||
17
.github/actions/setup-backend/action.yml
vendored
17
.github/actions/setup-backend/action.yml
vendored
@@ -28,7 +28,6 @@ runs:
|
||||
if [ "${{ inputs.python-version }}" = "current" ]; then
|
||||
echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
|
||||
elif [ "${{ inputs.python-version }}" = "next" ]; then
|
||||
# currently disabled in GHA matrixes because of library compatibility issues
|
||||
echo "PYTHON_VERSION=3.12" >> $GITHUB_ENV
|
||||
elif [ "${{ inputs.python-version }}" = "previous" ]; then
|
||||
echo "PYTHON_VERSION=3.10" >> $GITHUB_ENV
|
||||
@@ -40,7 +39,17 @@ runs:
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: ${{ inputs.cache }}
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ runner.os }}-python${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements/development.txt', 'requirements/base.txt') }}
|
||||
restore-keys: |
|
||||
uv-${{ runner.os }}-python${{ env.PYTHON_VERSION }}-
|
||||
- name: Install dependencies
|
||||
env:
|
||||
UV_CACHE_DIR: ~/.cache/uv
|
||||
UV_PREFER_BINARY: "1"
|
||||
run: |
|
||||
if [ "${{ inputs.install-superset }}" = "true" ]; then
|
||||
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
|
||||
@@ -48,11 +57,11 @@ runs:
|
||||
pip install --upgrade pip setuptools wheel uv
|
||||
|
||||
if [ "${{ inputs.requirements-type }}" = "dev" ]; then
|
||||
uv pip install --system -r requirements/development.txt
|
||||
uv pip install --system --prefer-binary -r requirements/development.txt
|
||||
elif [ "${{ inputs.requirements-type }}" = "base" ]; then
|
||||
uv pip install --system -r requirements/base.txt
|
||||
uv pip install --system --prefer-binary -r requirements/base.txt
|
||||
fi
|
||||
|
||||
uv pip install --system -e .
|
||||
uv pip install --system --prefer-binary -e .
|
||||
fi
|
||||
shell: bash
|
||||
|
||||
1
.github/copilot-instructions.md
vendored
Symbolic link
1
.github/copilot-instructions.md
vendored
Symbolic link
@@ -0,0 +1 @@
|
||||
../LLMS.md
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -127,5 +127,7 @@ docker/*local*
|
||||
# Jest test report
|
||||
test-report.html
|
||||
superset/static/stats/statistics.html
|
||||
.aider*
|
||||
|
||||
# LLM-related
|
||||
CLAUDE.local.md
|
||||
.aider*
|
||||
|
||||
@@ -76,3 +76,11 @@ ydb.svg
|
||||
erd.puml
|
||||
erd.svg
|
||||
intro_header.txt
|
||||
|
||||
# for LLMs
|
||||
llm-context.md
|
||||
LLMS.md
|
||||
CLAUDE.md
|
||||
CURSOR.md
|
||||
GEMINI.md
|
||||
GPT.md
|
||||
|
||||
@@ -167,7 +167,7 @@ RUN mkdir -p \
|
||||
&& touch superset/static/version_info.json
|
||||
|
||||
# Install Playwright and optionally setup headless browsers
|
||||
ARG INCLUDE_CHROMIUM="true"
|
||||
ARG INCLUDE_CHROMIUM="false"
|
||||
ARG INCLUDE_FIREFOX="false"
|
||||
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
|
||||
if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
|
||||
@@ -223,7 +223,7 @@ RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
|
||||
/app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
|
||||
# Install the superset package
|
||||
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
|
||||
uv pip install .
|
||||
uv pip install -e .
|
||||
RUN python -m compileall /app/superset
|
||||
|
||||
USER superset
|
||||
@@ -246,7 +246,7 @@ RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
|
||||
/app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt
|
||||
# Install the superset package
|
||||
RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \
|
||||
uv pip install .
|
||||
uv pip install -e .
|
||||
|
||||
RUN uv pip install .[postgres]
|
||||
RUN python -m compileall /app/superset
|
||||
|
||||
148
LLMS.md
Normal file
148
LLMS.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# LLM Context Guide for Apache Superset
|
||||
|
||||
Apache Superset is a data visualization platform with Flask/Python backend and React/TypeScript frontend.
|
||||
|
||||
## ⚠️ CRITICAL: Ongoing Refactors (What NOT to Do)
|
||||
|
||||
**These migrations are actively happening - avoid deprecated patterns:**
|
||||
|
||||
### Frontend Modernization
|
||||
- **NO `any` types** - Use proper TypeScript types
|
||||
- **NO JavaScript files** - Convert to TypeScript (.ts/.tsx)
|
||||
- **Use @superset-ui/core** - Don't import Ant Design directly
|
||||
|
||||
### Testing Strategy Migration
|
||||
- **Prefer unit tests** over integration tests
|
||||
- **Prefer integration tests** over Cypress end-to-end tests
|
||||
- **Cypress is last resort** - Actively moving away from Cypress
|
||||
- **Use Jest + React Testing Library** for component testing
|
||||
|
||||
### Backend Type Safety
|
||||
- **Add type hints** - All new Python code needs proper typing
|
||||
- **MyPy compliance** - Run `pre-commit run mypy` to validate
|
||||
- **SQLAlchemy typing** - Use proper model annotations
|
||||
|
||||
## Key Directories
|
||||
|
||||
```
|
||||
superset/
|
||||
├── superset/ # Python backend (Flask, SQLAlchemy)
|
||||
│ ├── views/api/ # REST API endpoints
|
||||
│ ├── models/ # Database models
|
||||
│ └── connectors/ # Database connections
|
||||
├── superset-frontend/src/ # React TypeScript frontend
|
||||
│ ├── components/ # Reusable components
|
||||
│ ├── explore/ # Chart builder
|
||||
│ ├── dashboard/ # Dashboard interface
|
||||
│ └── SqlLab/ # SQL editor
|
||||
├── superset-frontend/packages/
|
||||
│ └── superset-ui-core/ # UI component library (USE THIS)
|
||||
├── tests/ # Python/integration tests
|
||||
├── docs/ # Documentation (UPDATE FOR CHANGES)
|
||||
└── UPDATING.md # Breaking changes log
|
||||
```
|
||||
|
||||
## Code Standards
|
||||
|
||||
### TypeScript Frontend
|
||||
- **Avoid `any` types** - Use proper TypeScript, reuse existing types
|
||||
- **Functional components** with hooks
|
||||
- **@superset-ui/core** for UI components (not direct antd)
|
||||
- **Jest** for testing (NO Enzyme)
|
||||
- **Redux** for global state where it exists, hooks for local
|
||||
|
||||
### Python Backend
|
||||
- **Type hints required** for all new code
|
||||
- **MyPy compliant** - run `pre-commit run mypy`
|
||||
- **SQLAlchemy models** with proper typing
|
||||
- **pytest** for testing
|
||||
|
||||
### Apache License Headers
|
||||
- **New files require ASF license headers** - When creating new code files, include the standard Apache Software Foundation license header
|
||||
- **LLM instruction files are excluded** - Files like LLMS.md, CLAUDE.md, etc. are in `.rat-excludes` to avoid header token overhead
|
||||
|
||||
## Documentation Requirements
|
||||
|
||||
- **docs/**: Update for any user-facing changes
|
||||
- **UPDATING.md**: Add breaking changes here
|
||||
- **Docstrings**: Required for new functions/classes
|
||||
|
||||
## Architecture Patterns
|
||||
|
||||
### Security & Features
|
||||
- **RBAC**: Role-based access via Flask-AppBuilder
|
||||
- **Feature flags**: Control feature rollouts
|
||||
- **Row-level security**: SQL-based data access control
|
||||
|
||||
## Test Utilities
|
||||
|
||||
### Python Test Helpers
|
||||
- **`SupersetTestCase`** - Base class in `tests/integration_tests/base_tests.py`
|
||||
- **`@with_config`** - Config mocking decorator
|
||||
- **`@with_feature_flags`** - Feature flag testing
|
||||
- **`login_as()`, `login_as_admin()`** - Authentication helpers
|
||||
- **`create_dashboard()`, `create_slice()`** - Data setup utilities
|
||||
|
||||
### TypeScript Test Helpers
|
||||
- **`superset-frontend/spec/helpers/testing-library.tsx`** - Custom render() with providers
|
||||
- **`createWrapper()`** - Redux/Router/Theme wrapper
|
||||
- **`selectOption()`** - Select component helper
|
||||
- **React Testing Library** - NO Enzyme (removed)
|
||||
|
||||
### Running Tests
|
||||
```bash
|
||||
# Frontend
|
||||
npm run test # All tests
|
||||
npm run test -- filename.test.tsx # Single file
|
||||
|
||||
# Backend
|
||||
pytest # All tests
|
||||
pytest tests/unit_tests/specific_test.py # Single file
|
||||
pytest tests/unit_tests/ # Directory
|
||||
|
||||
# If pytest fails with database/setup issues, ask the user to run test environment setup
|
||||
```
|
||||
|
||||
## Environment Validation
|
||||
|
||||
**Quick Setup Check (run this first):**
|
||||
|
||||
```bash
|
||||
# Verify Superset is running
|
||||
curl -f http://localhost:8088/health || echo "❌ Setup required - see https://superset.apache.org/docs/contributing/development#working-with-llms"
|
||||
```
|
||||
|
||||
**If health checks fail:**
|
||||
"It appears you aren't set up properly. Please refer to the [Working with LLMs](https://superset.apache.org/docs/contributing/development#working-with-llms) section in the development docs for setup instructions."
|
||||
|
||||
**Key Project Files:**
|
||||
- `superset-frontend/package.json` - Frontend build scripts (`npm run dev` on port 9000, `npm run test`, `npm run lint`)
|
||||
- `pyproject.toml` - Python tooling (ruff, mypy configs)
|
||||
- `requirements/` folder - Python dependencies (base.txt, development.txt)
|
||||
|
||||
## Pre-commit Validation
|
||||
|
||||
**Use pre-commit hooks for quality validation:**
|
||||
|
||||
```bash
|
||||
# Install hooks
|
||||
pre-commit install
|
||||
|
||||
# Quick validation (faster than --all-files)
|
||||
pre-commit run # Staged files only
|
||||
pre-commit run mypy # Python type checking
|
||||
pre-commit run prettier # Code formatting
|
||||
pre-commit run eslint # Frontend linting
|
||||
```
|
||||
|
||||
## Platform-Specific Instructions
|
||||
|
||||
- **[CLAUDE.md](CLAUDE.md)** - For Claude/Anthropic tools
|
||||
- **[.github/copilot-instructions.md](.github/copilot-instructions.md)** - For GitHub Copilot
|
||||
- **[GEMINI.md](GEMINI.md)** - For Google Gemini tools
|
||||
- **[GPT.md](GPT.md)** - For OpenAI/ChatGPT tools
|
||||
- **[.cursor/rules/dev-standard.mdc](.cursor/rules/dev-standard.mdc)** - For Cursor editor
|
||||
|
||||
---
|
||||
|
||||
**LLM Note**: This codebase is actively modernizing toward full TypeScript and type safety. Always run `pre-commit run` to validate changes. Follow the ongoing refactors section to avoid deprecated patterns.
|
||||
@@ -23,6 +23,7 @@ This file documents any backwards-incompatible changes in Superset and
|
||||
assists people when migrating to a new version.
|
||||
|
||||
## Next
|
||||
- [34258](https://github.com/apache/superset/pull/34258) changing the default in Dockerfile to INCLUDE_CHROMIUM="false" (from "true") in the past. This ensures the `lean` layer is lean by default, and people can opt-in to the `chromium` layer by setting the build arg `INCLUDE_CHROMIUM=true`. This is a breaking change for anyone using the `lean` layer, as it will no longer include Chromium by default.
|
||||
- [34204](https://github.com/apache/superset/pull/33603) OpenStreetView has been promoted as the new default for Deck.gl visualization since it can be enabled by default without requiring an API key. If you have Mapbox set up and want to disable OpenStreeView in your environment, please follow the steps documented here [https://superset.apache.org/docs/configuration/map-tiles].
|
||||
- [33116](https://github.com/apache/superset/pull/33116) In Echarts Series charts (e.g. Line, Area, Bar, etc.) charts, the `x_axis_sort_series` and `x_axis_sort_series_ascending` form data items have been renamed with `x_axis_sort` and `x_axis_sort_asc`.
|
||||
There's a migration added that can potentially affect a significant number of existing charts.
|
||||
|
||||
@@ -194,6 +194,48 @@ You can also run the pre-commit checks manually in various ways:
|
||||
Replace `<hook_id>` with the ID of the specific hook you want to run. You can find the list
|
||||
of available hooks in the `.pre-commit-config.yaml` file.
|
||||
|
||||
## Working with LLMs
|
||||
|
||||
### Environment Setup
|
||||
Ensure Docker Compose is running before starting LLM sessions:
|
||||
```bash
|
||||
docker compose up
|
||||
```
|
||||
|
||||
Validate your environment:
|
||||
```bash
|
||||
curl -f http://localhost:8088/health && echo "✅ Superset ready"
|
||||
```
|
||||
|
||||
### LLM Session Best Practices
|
||||
- Always validate environment setup first using the health checks above
|
||||
- Use focused validation commands: `pre-commit run` (not `--all-files`)
|
||||
- **Read [LLMS.md](https://github.com/apache/superset/blob/master/LLMS.md) first** - Contains comprehensive development guidelines, coding standards, and critical refactor information
|
||||
- **Check platform-specific files** when available:
|
||||
- `CLAUDE.md` - For Claude/Anthropic tools
|
||||
- `CURSOR.md` - For Cursor editor
|
||||
- `GEMINI.md` - For Google Gemini tools
|
||||
- `GPT.md` - For OpenAI/ChatGPT tools
|
||||
- Follow the TypeScript migration guidelines and avoid deprecated patterns listed in LLMS.md
|
||||
|
||||
### Key Development Commands
|
||||
```bash
|
||||
# Frontend development
|
||||
cd superset-frontend
|
||||
npm run dev # Development server on http://localhost:9000
|
||||
npm run test # Run all tests
|
||||
npm run test -- filename.test.tsx # Run single test file
|
||||
npm run lint # Linting and type checking
|
||||
|
||||
# Backend validation
|
||||
pre-commit run mypy # Type checking
|
||||
pytest # Run all tests
|
||||
pytest tests/unit_tests/specific_test.py # Run single test file
|
||||
pytest tests/unit_tests/ # Run all tests in directory
|
||||
```
|
||||
|
||||
For detailed development context, environment setup, and coding guidelines, see [LLMS.md](https://github.com/apache/superset/blob/master/LLMS.md).
|
||||
|
||||
## Alternatives to `docker compose`
|
||||
|
||||
:::caution
|
||||
|
||||
@@ -202,7 +202,7 @@ export function AsyncAceEditor(
|
||||
|
||||
/* Basic editor styles with dark mode support */
|
||||
.ace_editor.ace-github,
|
||||
.ace_editor.ace-textmate {
|
||||
.ace_editor.ace-tm {
|
||||
background-color: ${token.colorBgContainer} !important;
|
||||
color: ${token.colorText} !important;
|
||||
}
|
||||
|
||||
@@ -70,7 +70,6 @@ export function Button(props: ButtonProps) {
|
||||
if (!buttonStyle || buttonStyle === 'primary') {
|
||||
variant = 'solid';
|
||||
antdType = 'primary';
|
||||
color = 'primary';
|
||||
} else if (buttonStyle === 'secondary') {
|
||||
variant = 'filled';
|
||||
color = 'primary';
|
||||
@@ -78,7 +77,6 @@ export function Button(props: ButtonProps) {
|
||||
variant = 'outlined';
|
||||
color = 'default';
|
||||
} else if (buttonStyle === 'dashed') {
|
||||
color = 'primary';
|
||||
variant = 'dashed';
|
||||
antdType = 'dashed';
|
||||
} else if (buttonStyle === 'danger') {
|
||||
@@ -134,6 +132,11 @@ export function Button(props: ButtonProps) {
|
||||
'& > span > :first-of-type': {
|
||||
marginRight: firstChildMargin,
|
||||
},
|
||||
':not(:hover)': effectiveButtonStyle === 'secondary' && {
|
||||
// NOTE: This is the best we can do contrast wise for the secondary button using antd tokens
|
||||
// and abusing the semantics. Should be revisited when possible. https://github.com/apache/superset/pull/34253#issuecomment-3104834692
|
||||
color: `${theme.colorPrimaryTextHover} !important`,
|
||||
},
|
||||
}}
|
||||
icon={icon}
|
||||
{...restProps}
|
||||
|
||||
@@ -31,11 +31,6 @@ const StyledDiv = styled.div`
|
||||
}
|
||||
`;
|
||||
|
||||
const DescriptionContainer = styled.div`
|
||||
line-height: ${({ theme }) => theme.sizeUnit * 4}px;
|
||||
padding-top: 16px;
|
||||
`;
|
||||
|
||||
export function DeleteModal({
|
||||
description,
|
||||
onConfirm,
|
||||
@@ -81,12 +76,12 @@ export function DeleteModal({
|
||||
onHide={hide}
|
||||
onHandledPrimaryAction={confirm}
|
||||
primaryButtonName={t('Delete')}
|
||||
primaryButtonType="danger"
|
||||
primaryButtonStyle="danger"
|
||||
show={open}
|
||||
title={title}
|
||||
centered
|
||||
>
|
||||
<DescriptionContainer>{description}</DescriptionContainer>
|
||||
{description}
|
||||
<StyledDiv>
|
||||
<FormLabel htmlFor="delete">
|
||||
{t('Type "%s" to confirm', t('DELETE'))}
|
||||
|
||||
@@ -33,7 +33,7 @@ export const InteractiveModal = (props: ModalProps) => (
|
||||
InteractiveModal.args = {
|
||||
disablePrimaryButton: false,
|
||||
primaryButtonName: 'Danger',
|
||||
primaryButtonType: 'danger',
|
||||
primaryButtonStyle: 'danger',
|
||||
show: true,
|
||||
title: "I'm a modal!",
|
||||
resizable: false,
|
||||
|
||||
@@ -77,7 +77,7 @@ export const StyledModal = styled(BaseModal)<StyledModalProps>`
|
||||
.ant-modal-header {
|
||||
flex: 0 0 auto;
|
||||
border-radius: ${theme.borderRadius}px ${theme.borderRadius}px 0 0;
|
||||
padding: ${theme.sizeUnit * 4}px ${theme.sizeUnit * 6}px;
|
||||
padding: ${theme.sizeUnit * 4}px ${theme.sizeUnit * 4}px;
|
||||
|
||||
.ant-modal-title {
|
||||
font-weight: ${theme.fontWeightStrong};
|
||||
@@ -122,6 +122,7 @@ export const StyledModal = styled(BaseModal)<StyledModalProps>`
|
||||
.ant-modal-body {
|
||||
flex: 0 1 auto;
|
||||
padding: ${theme.sizeUnit * 4}px;
|
||||
padding-bottom: ${theme.sizeUnit * 2}px;
|
||||
overflow: auto;
|
||||
${!resizable && height && `height: ${height};`}
|
||||
}
|
||||
@@ -208,7 +209,7 @@ const CustomModal = ({
|
||||
onHide,
|
||||
onHandledPrimaryAction,
|
||||
primaryButtonName = t('OK'),
|
||||
primaryButtonType = 'primary',
|
||||
primaryButtonStyle = 'primary',
|
||||
show,
|
||||
name,
|
||||
title,
|
||||
@@ -261,7 +262,7 @@ const CustomModal = ({
|
||||
</Button>,
|
||||
<Button
|
||||
key="submit"
|
||||
buttonStyle={primaryButtonType}
|
||||
buttonStyle={primaryButtonStyle}
|
||||
disabled={disablePrimaryButton}
|
||||
tooltip={primaryTooltipMessage}
|
||||
loading={primaryButtonLoading}
|
||||
|
||||
@@ -20,6 +20,7 @@ import type { CSSProperties, ReactNode } from 'react';
|
||||
import type { ModalFuncProps } from 'antd';
|
||||
import type { ResizableProps } from 're-resizable';
|
||||
import type { DraggableProps } from 'react-draggable';
|
||||
import { ButtonStyle } from '../Button/types';
|
||||
|
||||
export interface ModalProps {
|
||||
className?: string;
|
||||
@@ -30,7 +31,7 @@ export interface ModalProps {
|
||||
onHide: () => void;
|
||||
onHandledPrimaryAction?: () => void;
|
||||
primaryButtonName?: string;
|
||||
primaryButtonType?: 'primary' | 'danger';
|
||||
primaryButtonStyle?: ButtonStyle;
|
||||
show: boolean;
|
||||
name?: string;
|
||||
title: ReactNode;
|
||||
|
||||
@@ -24,7 +24,7 @@ import {
|
||||
getSequentialSchemeRegistry,
|
||||
CategoricalColorNamespace,
|
||||
} from '@superset-ui/core';
|
||||
import Datamap from 'datamaps/dist/datamaps.world.min';
|
||||
import Datamap from 'datamaps/dist/datamaps.all.min';
|
||||
import { ColorBy } from './utils';
|
||||
|
||||
const propTypes = {
|
||||
|
||||
@@ -54,7 +54,8 @@ export function ErrorMessageWithStackTrace({
|
||||
// Check if a custom error message component was registered for this message
|
||||
if (error) {
|
||||
const ErrorMessageComponent = getErrorMessageComponentRegistry().get(
|
||||
error.error_type,
|
||||
// @ts-ignore: plan to modify this part so that all errors in Superset 6.0 are standardized as Superset API error types
|
||||
error.errorType ?? error.error_type,
|
||||
);
|
||||
if (ErrorMessageComponent) {
|
||||
return (
|
||||
|
||||
@@ -362,7 +362,7 @@ export const ImportModal: FunctionComponent<ImportModelsModalProps> = ({
|
||||
onHandledPrimaryAction={onUpload}
|
||||
onHide={hide}
|
||||
primaryButtonName={needsOverwriteConfirm ? t('Overwrite') : t('Import')}
|
||||
primaryButtonType={needsOverwriteConfirm ? 'danger' : 'primary'}
|
||||
primaryButtonStyle={needsOverwriteConfirm ? 'danger' : 'primary'}
|
||||
width="750px"
|
||||
show={show}
|
||||
title={<h4>{t('Import %s', resourceLabel)}</h4>}
|
||||
|
||||
@@ -129,6 +129,9 @@ const Chart = props => {
|
||||
);
|
||||
|
||||
const chart = useSelector(state => state.charts[props.id] || EMPTY_OBJECT);
|
||||
const { queriesResponse, chartUpdateEndTime, chartStatus, annotationQuery } =
|
||||
chart;
|
||||
|
||||
const slice = useSelector(
|
||||
state => state.sliceEntities.slices[props.id] || EMPTY_OBJECT,
|
||||
);
|
||||
@@ -163,6 +166,12 @@ const Chart = props => {
|
||||
);
|
||||
const dashboardInfo = useSelector(state => state.dashboardInfo);
|
||||
|
||||
const isCached = useMemo(
|
||||
// eslint-disable-next-line camelcase
|
||||
() => queriesResponse?.map(({ is_cached }) => is_cached) || [],
|
||||
[queriesResponse],
|
||||
);
|
||||
|
||||
const [descriptionHeight, setDescriptionHeight] = useState(0);
|
||||
const [height, setHeight] = useState(props.height);
|
||||
const [width, setWidth] = useState(props.width);
|
||||
@@ -249,9 +258,9 @@ const Chart = props => {
|
||||
const logExploreChart = useCallback(() => {
|
||||
boundActionCreators.logEvent(LOG_ACTIONS_EXPLORE_DASHBOARD_CHART, {
|
||||
slice_id: slice.slice_id,
|
||||
is_cached: props.isCached,
|
||||
is_cached: isCached,
|
||||
});
|
||||
}, [boundActionCreators.logEvent, slice.slice_id, props.isCached]);
|
||||
}, [boundActionCreators.logEvent, slice.slice_id, isCached]);
|
||||
|
||||
const chartConfiguration = useSelector(
|
||||
state => state.dashboardInfo.metadata?.chart_configuration,
|
||||
@@ -365,22 +374,22 @@ const Chart = props => {
|
||||
: LOG_ACTIONS_EXPORT_XLSX_DASHBOARD_CHART;
|
||||
boundActionCreators.logEvent(logAction, {
|
||||
slice_id: slice.slice_id,
|
||||
is_cached: props.isCached,
|
||||
is_cached: isCached,
|
||||
});
|
||||
exportChart({
|
||||
formData: isFullCSV ? { ...formData, row_limit: maxRows } : formData,
|
||||
resultType: isPivot ? 'post_processed' : 'full',
|
||||
resultFormat: format,
|
||||
force: true,
|
||||
ownState: props.ownState,
|
||||
ownState: dataMask[props.id]?.ownState,
|
||||
});
|
||||
},
|
||||
[
|
||||
slice.slice_id,
|
||||
props.isCached,
|
||||
isCached,
|
||||
formData,
|
||||
props.maxRows,
|
||||
props.ownState,
|
||||
dataMask[props.id]?.ownState,
|
||||
boundActionCreators.logEvent,
|
||||
],
|
||||
);
|
||||
@@ -408,7 +417,7 @@ const Chart = props => {
|
||||
const forceRefresh = useCallback(() => {
|
||||
boundActionCreators.logEvent(LOG_ACTIONS_FORCE_REFRESH_CHART, {
|
||||
slice_id: slice.slice_id,
|
||||
is_cached: props.isCached,
|
||||
is_cached: isCached,
|
||||
});
|
||||
return boundActionCreators.refreshChart(chart.id, true, props.dashboardId);
|
||||
}, [
|
||||
@@ -416,7 +425,7 @@ const Chart = props => {
|
||||
chart.id,
|
||||
props.dashboardId,
|
||||
slice.slice_id,
|
||||
props.isCached,
|
||||
isCached,
|
||||
boundActionCreators.logEvent,
|
||||
]);
|
||||
|
||||
@@ -424,11 +433,7 @@ const Chart = props => {
|
||||
return <MissingChart height={getChartHeight()} />;
|
||||
}
|
||||
|
||||
const { queriesResponse, chartUpdateEndTime, chartStatus, annotationQuery } =
|
||||
chart;
|
||||
const isLoading = chartStatus === 'loading';
|
||||
// eslint-disable-next-line camelcase
|
||||
const isCached = queriesResponse?.map(({ is_cached }) => is_cached) || [];
|
||||
const cachedDttm =
|
||||
// eslint-disable-next-line camelcase
|
||||
queriesResponse?.map(({ cached_dttm }) => cached_dttm) || [];
|
||||
|
||||
@@ -46,7 +46,7 @@ const containerStyle = (theme: SupersetTheme) => css`
|
||||
display: flex;
|
||||
|
||||
&& > .filter-clear-all-button {
|
||||
color: ${theme.colors.grayscale.base};
|
||||
color: ${theme.colorTextSecondary};
|
||||
margin-left: 0;
|
||||
&:hover {
|
||||
color: ${theme.colorPrimaryText};
|
||||
@@ -54,7 +54,7 @@ const containerStyle = (theme: SupersetTheme) => css`
|
||||
|
||||
&[disabled],
|
||||
&[disabled]:hover {
|
||||
color: ${theme.colors.grayscale.light1};
|
||||
color: ${theme.colorTextDisabled};
|
||||
}
|
||||
}
|
||||
`;
|
||||
@@ -62,7 +62,6 @@ const containerStyle = (theme: SupersetTheme) => css`
|
||||
const verticalStyle = (theme: SupersetTheme, width: number) => css`
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
pointer-events: none;
|
||||
position: fixed;
|
||||
z-index: 100;
|
||||
|
||||
@@ -74,14 +73,10 @@ const verticalStyle = (theme: SupersetTheme, width: number) => css`
|
||||
padding-top: ${theme.sizeUnit * 6}px;
|
||||
|
||||
background: linear-gradient(
|
||||
${rgba(theme.colors.grayscale.light5, 0)},
|
||||
${theme.colors.grayscale.light5} 60%
|
||||
${rgba(theme.colorBgLayout, 0)},
|
||||
${theme.colorBgElevated} 20%
|
||||
);
|
||||
|
||||
& > button {
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
& > .filter-apply-button {
|
||||
margin-bottom: ${theme.sizeUnit * 3}px;
|
||||
}
|
||||
@@ -94,13 +89,6 @@ const horizontalStyle = (theme: SupersetTheme) => css`
|
||||
text-transform: capitalize;
|
||||
font-weight: ${theme.fontWeightNormal};
|
||||
}
|
||||
& > .filter-apply-button {
|
||||
&[disabled],
|
||||
&[disabled]:hover {
|
||||
color: ${theme.colors.grayscale.light1};
|
||||
background: ${theme.colors.grayscale.light3};
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const ButtonsContainer = styled.div<{ isVertical: boolean; width: number }>`
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
|
||||
import { Icons } from '@superset-ui/core/components/Icons';
|
||||
import { Tooltip } from '@superset-ui/core/components/Tooltip';
|
||||
import { Typography } from '@superset-ui/core/components';
|
||||
import DatasourcePanelDragOption from './DatasourcePanelDragOption';
|
||||
import { DndItemType } from '../DndItemType';
|
||||
import { DndItemValue, FlattenedItem, Folder } from './types';
|
||||
@@ -94,7 +95,7 @@ const SectionHeaderTextContainer = styled.div`
|
||||
width: 100%;
|
||||
`;
|
||||
|
||||
const SectionHeader = styled.span`
|
||||
const SectionHeader = styled(Typography.Text)`
|
||||
${({ theme }) => css`
|
||||
font-size: ${theme.fontSize}px;
|
||||
font-weight: ${theme.fontWeightStrong};
|
||||
|
||||
@@ -95,6 +95,7 @@ const Styles = styled.div`
|
||||
}
|
||||
.error-alert {
|
||||
margin: ${({ theme }) => 2 * theme.sizeUnit}px;
|
||||
min-height: 150px;
|
||||
}
|
||||
.ant-dropdown-trigger {
|
||||
margin-left: ${({ theme }) => 2 * theme.sizeUnit}px;
|
||||
@@ -454,16 +455,14 @@ class DatasourceControl extends PureComponent {
|
||||
{isMissingDatasource && !isMissingParams && (
|
||||
<div className="error-alert">
|
||||
{extra?.error ? (
|
||||
<div className="error-alert">
|
||||
<ErrorMessageWithStackTrace
|
||||
title={extra.error.statusText || extra.error.message}
|
||||
subtitle={
|
||||
extra.error.statusText ? extra.error.message : undefined
|
||||
}
|
||||
error={extra.error}
|
||||
source="explore"
|
||||
/>
|
||||
</div>
|
||||
<ErrorMessageWithStackTrace
|
||||
title={extra.error.statusText || extra.error.message}
|
||||
subtitle={
|
||||
extra.error.statusText ? extra.error.message : undefined
|
||||
}
|
||||
error={extra.error}
|
||||
source="explore"
|
||||
/>
|
||||
) : (
|
||||
<ErrorAlert
|
||||
type="warning"
|
||||
|
||||
@@ -102,9 +102,9 @@ const getCommonElements = () => ({
|
||||
cancelButton: screen.getByRole('button', { name: 'Cancel' }),
|
||||
uploadButton: screen.getByRole('button', { name: 'Upload' }),
|
||||
selectButton: screen.getByRole('button', { name: 'Select' }),
|
||||
panel1: screen.getByRole('heading', { name: /General information/i }),
|
||||
panel2: screen.getByRole('heading', { name: /file settings/i }),
|
||||
panel3: screen.getByRole('heading', { name: /columns/i }),
|
||||
panel1: screen.getByText(/General information/i, { selector: 'strong' }),
|
||||
panel2: screen.getByText(/file settings/i, { selector: 'strong' }),
|
||||
panel3: screen.getByText(/columns/i, { selector: 'strong' }),
|
||||
selectDatabase: screen.getByRole('combobox', { name: /select a database/i }),
|
||||
inputTableName: screen.getByRole('textbox', { name: /table name/i }),
|
||||
inputSchema: screen.getByRole('combobox', { name: /schema/i }),
|
||||
@@ -130,7 +130,7 @@ describe('UploadDataModal - General Information Elements', () => {
|
||||
|
||||
const common = getCommonElements();
|
||||
const title = screen.getByRole('heading', { name: /csv upload/i });
|
||||
const panel4 = screen.getByRole('heading', { name: /rows/i });
|
||||
const panel4 = screen.getByText(/rows/i);
|
||||
const selectDelimiter = screen.getByRole('combobox', {
|
||||
name: /choose a delimiter/i,
|
||||
});
|
||||
@@ -156,7 +156,7 @@ describe('UploadDataModal - General Information Elements', () => {
|
||||
|
||||
const common = getCommonElements();
|
||||
const title = screen.getByRole('heading', { name: /excel upload/i });
|
||||
const panel4 = screen.getByRole('heading', { name: /rows/i });
|
||||
const panel4 = screen.getByText(/rows/i);
|
||||
const selectSheetName = screen.getByRole('combobox', {
|
||||
name: /choose sheet name/i,
|
||||
});
|
||||
@@ -177,9 +177,7 @@ describe('UploadDataModal - General Information Elements', () => {
|
||||
]);
|
||||
|
||||
// Check elements that should NOT be visible
|
||||
expect(
|
||||
screen.queryByRole('heading', { name: /csv upload/i }),
|
||||
).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/csv upload/i)).not.toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByRole('combobox', { name: /choose a delimiter/i }),
|
||||
).not.toBeInTheDocument();
|
||||
@@ -206,8 +204,8 @@ describe('UploadDataModal - General Information Elements', () => {
|
||||
|
||||
// Check elements that should NOT be visible
|
||||
expectElementsNotVisible([
|
||||
screen.queryByRole('heading', { name: /csv upload/i }),
|
||||
screen.queryByRole('heading', { name: /rows/i }),
|
||||
screen.queryByText(/csv upload/i),
|
||||
screen.queryByText(/rows/i),
|
||||
screen.queryByRole('combobox', { name: /choose a delimiter/i }),
|
||||
screen.queryByRole('combobox', { name: /choose sheet name/i }),
|
||||
]);
|
||||
@@ -216,7 +214,7 @@ describe('UploadDataModal - General Information Elements', () => {
|
||||
|
||||
describe('UploadDataModal - File Settings Elements', () => {
|
||||
const openFileSettings = async () => {
|
||||
const panelHeader = screen.getByRole('heading', { name: /file settings/i });
|
||||
const panelHeader = screen.getByText(/file settings/i);
|
||||
await userEvent.click(panelHeader);
|
||||
};
|
||||
|
||||
@@ -294,7 +292,7 @@ describe('UploadDataModal - File Settings Elements', () => {
|
||||
|
||||
describe('UploadDataModal - Columns Elements', () => {
|
||||
const openColumns = async () => {
|
||||
const panelHeader = screen.getByRole('heading', { name: /columns/i });
|
||||
const panelHeader = screen.getByText(/columns/i, { selector: 'strong' });
|
||||
await userEvent.click(panelHeader);
|
||||
};
|
||||
|
||||
@@ -365,7 +363,7 @@ describe('UploadDataModal - Rows Elements', () => {
|
||||
test('CSV/Excel rows render correctly', async () => {
|
||||
render(<UploadDataModal {...csvProps} />, { useRedux: true });
|
||||
|
||||
const panelHeader = screen.getByRole('heading', { name: /rows/i });
|
||||
const panelHeader = screen.getByText(/rows/i);
|
||||
await userEvent.click(panelHeader);
|
||||
|
||||
const elements = [
|
||||
@@ -380,7 +378,7 @@ describe('UploadDataModal - Rows Elements', () => {
|
||||
test('Columnar does not render rows', () => {
|
||||
render(<UploadDataModal {...columnarProps} />, { useRedux: true });
|
||||
|
||||
const panelHeader = screen.queryByRole('heading', { name: /rows/i });
|
||||
const panelHeader = screen.queryByText(/rows/i);
|
||||
expect(panelHeader).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -608,11 +606,11 @@ describe('UploadDataModal Collapse Tabs', () => {
|
||||
useRedux: true,
|
||||
});
|
||||
const generalInfoTab = screen.getByRole('tab', {
|
||||
name: /expanded General information Upload a file to a database./i,
|
||||
name: /expanded General information/i,
|
||||
});
|
||||
expect(generalInfoTab).toHaveAttribute('aria-expanded', 'true');
|
||||
const fileSettingsTab = screen.getByRole('tab', {
|
||||
name: /collapsed File settings Adjust how spaces, blank lines, null values are handled and other file wide settings./i,
|
||||
name: /collapsed File settings/i,
|
||||
});
|
||||
userEvent.click(fileSettingsTab);
|
||||
expect(fileSettingsTab).toHaveAttribute('aria-expanded', 'true');
|
||||
@@ -626,11 +624,11 @@ describe('UploadDataModal Collapse Tabs', () => {
|
||||
useRedux: true,
|
||||
});
|
||||
const generalInfoTab = screen.getByRole('tab', {
|
||||
name: /expanded General information Upload a file to a database./i,
|
||||
name: /expanded General information/i,
|
||||
});
|
||||
expect(generalInfoTab).toHaveAttribute('aria-expanded', 'true');
|
||||
const fileSettingsTab = screen.getByRole('tab', {
|
||||
name: /collapsed File settings Adjust how spaces, blank lines, null values are handled and other file wide settings./i,
|
||||
name: /collapsed File settings/i,
|
||||
});
|
||||
userEvent.click(fileSettingsTab);
|
||||
expect(fileSettingsTab).toHaveAttribute('aria-expanded', 'true');
|
||||
@@ -644,11 +642,11 @@ describe('UploadDataModal Collapse Tabs', () => {
|
||||
useRedux: true,
|
||||
});
|
||||
const generalInfoTab = screen.getByRole('tab', {
|
||||
name: /expanded General information Upload a file to a database./i,
|
||||
name: /expanded General information/i,
|
||||
});
|
||||
expect(generalInfoTab).toHaveAttribute('aria-expanded', 'true');
|
||||
const fileSettingsTab = screen.getByRole('tab', {
|
||||
name: /collapsed File settings Adjust how spaces, blank lines, null values are handled and other file wide settings./i,
|
||||
name: /collapsed File settings/i,
|
||||
});
|
||||
userEvent.click(fileSettingsTab);
|
||||
expect(fileSettingsTab).toHaveAttribute('aria-expanded', 'true');
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
} from 'react';
|
||||
|
||||
import {
|
||||
css,
|
||||
getClientErrorObject,
|
||||
SupersetClient,
|
||||
SupersetTheme,
|
||||
@@ -45,6 +46,7 @@ import {
|
||||
Upload,
|
||||
type UploadChangeParam,
|
||||
type UploadFile,
|
||||
Typography,
|
||||
} from '@superset-ui/core/components';
|
||||
import { Switch, SwitchProps } from '@superset-ui/core/components/Switch';
|
||||
import { Icons } from '@superset-ui/core/components/Icons';
|
||||
@@ -576,7 +578,17 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
|
||||
const UploadTitle: FC = () => {
|
||||
const title = uploadTitles[type] || t('Upload');
|
||||
return <h4>{title}</h4>;
|
||||
return (
|
||||
<Typography.Title
|
||||
level={5}
|
||||
css={css`
|
||||
margin-top: 0;
|
||||
margin-bottom: 0;
|
||||
`}
|
||||
>
|
||||
{title}
|
||||
</Typography.Title>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -592,7 +604,7 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
onHandledPrimaryAction={form.submit}
|
||||
onHide={onClose}
|
||||
width="500px"
|
||||
primaryButtonName="Upload"
|
||||
primaryButtonName={t('Upload')}
|
||||
centered
|
||||
show={show}
|
||||
title={<UploadTitle />}
|
||||
@@ -615,10 +627,9 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
{
|
||||
key: 'general',
|
||||
label: (
|
||||
<div>
|
||||
<h4>{t('General information')}</h4>
|
||||
<p className="helper">{t('Upload a file to a database.')}</p>
|
||||
</div>
|
||||
<Typography.Text strong>
|
||||
{t('General information')}
|
||||
</Typography.Text>
|
||||
),
|
||||
children: (
|
||||
<>
|
||||
@@ -772,14 +783,7 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
{
|
||||
key: 'file-settings',
|
||||
label: (
|
||||
<div>
|
||||
<h4>{t('File settings')}</h4>
|
||||
<p className="helper">
|
||||
{t(
|
||||
'Adjust how spaces, blank lines, null values are handled and other file wide settings.',
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
<Typography.Text strong>{t('File settings')}</Typography.Text>
|
||||
),
|
||||
children: (
|
||||
<>
|
||||
@@ -901,16 +905,7 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
},
|
||||
{
|
||||
key: 'columns',
|
||||
label: (
|
||||
<div>
|
||||
<h4>{t('Columns')}</h4>
|
||||
<p className="helper">
|
||||
{t(
|
||||
'Adjust column settings such as specifying the columns to read, how duplicates are handled, column data types, and more.',
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
),
|
||||
label: <Typography.Text strong>{t('Columns')}</Typography.Text>,
|
||||
children: (
|
||||
<>
|
||||
<Row>
|
||||
@@ -1010,14 +1005,7 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
|
||||
{
|
||||
key: 'rows',
|
||||
label: (
|
||||
<div>
|
||||
<h4>{t('Rows')}</h4>
|
||||
<p className="helper">
|
||||
{t(
|
||||
'Set header rows and the number of rows to read or skip.',
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
<Typography.Text strong>{t('Rows')}</Typography.Text>
|
||||
),
|
||||
children: (
|
||||
<Row>
|
||||
|
||||
@@ -46,7 +46,7 @@ export const antDModalNoPaddingStyles = css`
|
||||
|
||||
export const formStyles = (theme: SupersetTheme) => css`
|
||||
.switch-label {
|
||||
color: ${theme.colors.grayscale.base};
|
||||
color: ${theme.colorTextSecondary};
|
||||
margin-left: ${theme.sizeUnit * 4}px;
|
||||
}
|
||||
`;
|
||||
|
||||
@@ -319,6 +319,13 @@ const config = {
|
||||
'./node_modules/@storybook/react-dom-shim/dist/react-16',
|
||||
),
|
||||
),
|
||||
// Workaround for react-color trying to import non-existent icon
|
||||
'@icons/material/UnfoldMoreHorizontalIcon': path.resolve(
|
||||
path.join(
|
||||
APP_DIR,
|
||||
'./node_modules/@icons/material/CubeUnfoldedIcon.js',
|
||||
),
|
||||
),
|
||||
},
|
||||
extensions: ['.ts', '.tsx', '.js', '.jsx', '.yml'],
|
||||
fallback: {
|
||||
|
||||
@@ -598,8 +598,8 @@ DEFAULT_FEATURE_FLAGS: dict[str, bool] = {
|
||||
# This is intended to use for theme creation, visual review and theming-debugging
|
||||
# purposes.
|
||||
"THEME_ALLOW_THEME_EDITOR_BETA": False,
|
||||
# Allow users to optionally specify date formats in email subjects, which will
|
||||
# be parsed if enabled
|
||||
# Allow users to optionally specify date formats in email subjects, which
|
||||
# will be parsed if enabled
|
||||
"DATE_FORMAT_IN_EMAIL_SUBJECT": False,
|
||||
# Allow metrics and columns to be grouped into (potentially nested) folders in the
|
||||
# chart builder
|
||||
|
||||
@@ -164,9 +164,7 @@ class DatasourceKind(StrEnum):
|
||||
PHYSICAL = "physical"
|
||||
|
||||
|
||||
class BaseDatasource(
|
||||
AuditMixinNullable, ImportExportMixin
|
||||
): # pylint: disable=too-many-public-methods
|
||||
class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=too-many-public-methods
|
||||
"""A common interface to objects that are queryable
|
||||
(tables and datasources)"""
|
||||
|
||||
@@ -1780,9 +1778,7 @@ class SqlaTable(
|
||||
def default_query(qry: Query) -> Query:
|
||||
return qry.filter_by(is_sqllab_view=False)
|
||||
|
||||
def has_extra_cache_key_calls(
|
||||
self, query_obj: QueryObjectDict
|
||||
) -> bool: # noqa: C901
|
||||
def has_extra_cache_key_calls(self, query_obj: QueryObjectDict) -> bool: # noqa: C901
|
||||
"""
|
||||
Detects the presence of calls to `ExtraCache` methods in items in query_obj that
|
||||
can be templated. If any are present, the query must be evaluated to extract
|
||||
|
||||
@@ -75,11 +75,12 @@ class DatasetDAO(BaseDAO[SqlaTable]):
|
||||
database: Database,
|
||||
table: Table,
|
||||
) -> bool:
|
||||
with database.get_inspector(
|
||||
catalog=table.catalog,
|
||||
schema=table.schema,
|
||||
) as inspector:
|
||||
return database.db_engine_spec.has_table(database, inspector, table)
|
||||
try:
|
||||
database.get_table(table)
|
||||
return True
|
||||
except SQLAlchemyError as ex: # pragma: no cover
|
||||
logger.warning("Got an error %s validating table: %s", str(ex), table)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def validate_uniqueness(
|
||||
|
||||
@@ -30,7 +30,6 @@ from typing import (
|
||||
cast,
|
||||
ContextManager,
|
||||
NamedTuple,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
TypedDict,
|
||||
Union,
|
||||
@@ -63,10 +62,6 @@ from superset.constants import QUERY_CANCEL_KEY, TimeGrain as TimeGrainConstants
|
||||
from superset.databases.utils import get_table_metadata, make_url_safe
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
from superset.exceptions import OAuth2Error, OAuth2RedirectError
|
||||
from superset.extensions.semantic_layer import (
|
||||
get_sqla_type_from_dimension_type,
|
||||
SemanticLayer,
|
||||
)
|
||||
from superset.sql.parse import (
|
||||
BaseSQLStatement,
|
||||
LimitMethod,
|
||||
@@ -90,7 +85,7 @@ from superset.utils.network import is_hostname_valid, is_port_open
|
||||
from superset.utils.oauth2 import encode_oauth2_state
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.connectors.sqla.models import SqlaTable, TableColumn
|
||||
from superset.connectors.sqla.models import TableColumn
|
||||
from superset.databases.schemas import TableMetadataResponse
|
||||
from superset.models.core import Database
|
||||
from superset.models.sql_lab import Query
|
||||
@@ -111,15 +106,6 @@ logger = logging.getLogger()
|
||||
GenericDBException = Exception
|
||||
|
||||
|
||||
class ValidColumnsType(TypedDict):
|
||||
"""
|
||||
Type for valid columns returned by `get_valid_metrics_and_dimensions`.
|
||||
"""
|
||||
|
||||
dimensions: set[str]
|
||||
metrics: set[str]
|
||||
|
||||
|
||||
def convert_inspector_columns(cols: list[SQLAColumnType]) -> list[ResultSetColumnType]:
|
||||
result_set_columns: list[ResultSetColumnType] = []
|
||||
for col in cols:
|
||||
@@ -157,9 +143,7 @@ builtin_time_grains: dict[str | None, str] = {
|
||||
}
|
||||
|
||||
|
||||
class TimestampExpression(
|
||||
ColumnClause
|
||||
): # pylint: disable=abstract-method, too-many-ancestors
|
||||
class TimestampExpression(ColumnClause): # pylint: disable=abstract-method, too-many-ancestors
|
||||
def __init__(self, expr: str, col: ColumnClause, **kwargs: Any) -> None:
|
||||
"""Sqlalchemy class that can be used to render native column elements respecting
|
||||
engine-specific quoting rules as part of a string-based expression.
|
||||
@@ -230,9 +214,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
"engine+driver://user:password@host:port/dbname[?key=value&key=value...]"
|
||||
)
|
||||
|
||||
# databases can optionally specify a semantic layer
|
||||
semantic_layer: Type[SemanticLayer] | None = None
|
||||
|
||||
disable_ssh_tunneling = False
|
||||
|
||||
_date_trunc_functions: dict[str, str] = {}
|
||||
@@ -407,9 +388,9 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
max_column_name_length: int | None = None
|
||||
try_remove_schema_from_table_name = True # pylint: disable=invalid-name
|
||||
run_multiple_statements_as_one = False
|
||||
custom_errors: dict[Pattern[str], tuple[str, SupersetErrorType, dict[str, Any]]] = (
|
||||
{}
|
||||
)
|
||||
custom_errors: dict[
|
||||
Pattern[str], tuple[str, SupersetErrorType, dict[str, Any]]
|
||||
] = {}
|
||||
|
||||
# List of JSON path to fields in `encrypted_extra` that should be masked when the
|
||||
# database is edited. By default everything is masked.
|
||||
@@ -1480,32 +1461,8 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
|
||||
if schema and cls.try_remove_schema_from_table_name:
|
||||
tables = {re.sub(f"^{schema}\\.", "", table) for table in tables}
|
||||
|
||||
# add semantic views as tables too
|
||||
if cls.semantic_layer:
|
||||
semantic_layer = cls.semantic_layer(inspector.engine)
|
||||
tables.update(
|
||||
semantic_view.name
|
||||
for semantic_view in semantic_layer.get_semantic_views()
|
||||
)
|
||||
|
||||
return tables
|
||||
|
||||
@classmethod
|
||||
def has_table(
|
||||
cls,
|
||||
database: Database,
|
||||
inspector: Inspector,
|
||||
table: Table,
|
||||
) -> bool:
|
||||
if cls.semantic_layer:
|
||||
semantic_layer = cls.semantic_layer(inspector.engine)
|
||||
semantic_views = semantic_layer.get_semantic_views()
|
||||
if table.table in {semantic_view.name for semantic_view in semantic_views}:
|
||||
return True
|
||||
|
||||
return inspector.has_table(table.table, table.schema)
|
||||
|
||||
@classmethod
|
||||
def get_view_names( # pylint: disable=unused-argument
|
||||
cls,
|
||||
@@ -1579,7 +1536,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
@classmethod
|
||||
def get_columns( # pylint: disable=unused-argument
|
||||
cls,
|
||||
database: Database,
|
||||
inspector: Inspector,
|
||||
table: Table,
|
||||
options: dict[str, Any] | None = None,
|
||||
@@ -1587,9 +1543,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
"""
|
||||
Get all columns from a given schema and table.
|
||||
|
||||
The inspector will be bound to a catalog, if one was specified. If the database
|
||||
supports semantic layers the method will check if the table is a semantic view,
|
||||
and return columns (metrics and dimensions) from it instead.
|
||||
The inspector will be bound to a catalog, if one was specified.
|
||||
|
||||
:param inspector: SqlAlchemy Inspector instance
|
||||
:param table: Table instance
|
||||
@@ -1597,26 +1551,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
some databases
|
||||
:return: All columns in table
|
||||
"""
|
||||
if cls.semantic_layer:
|
||||
semantic_layer = cls.semantic_layer(inspector.engine)
|
||||
semantic_views = {
|
||||
semantic_view.name: semantic_view
|
||||
for semantic_view in semantic_layer.get_semantic_views()
|
||||
}
|
||||
if semantic_view := semantic_views.get(table.table):
|
||||
dialect = database.get_dialect()
|
||||
return [
|
||||
{
|
||||
"name": dimension.name,
|
||||
"column_name": dimension.name,
|
||||
"type": cls.column_datatype_to_string(
|
||||
get_sqla_type_from_dimension_type(dimension.type),
|
||||
dialect,
|
||||
),
|
||||
}
|
||||
for dimension in semantic_layer.get_dimensions(semantic_view)
|
||||
]
|
||||
|
||||
return convert_inspector_columns(
|
||||
cast(
|
||||
list[SQLAColumnType],
|
||||
@@ -1634,22 +1568,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
"""
|
||||
Get all metrics from a given schema and table.
|
||||
"""
|
||||
if cls.semantic_layer:
|
||||
semantic_layer = cls.semantic_layer(inspector.engine)
|
||||
semantic_views = {
|
||||
semantic_view.name: semantic_view
|
||||
for semantic_view in semantic_layer.get_semantic_views()
|
||||
}
|
||||
if semantic_view := semantic_views.get(table.table):
|
||||
return [
|
||||
{
|
||||
"metric_name": metric.name,
|
||||
"verbose_name": metric.name,
|
||||
"expression": metric.sql,
|
||||
}
|
||||
for metric in semantic_layer.get_metrics(semantic_view)
|
||||
]
|
||||
|
||||
return [
|
||||
{
|
||||
"metric_name": "count",
|
||||
@@ -1659,62 +1577,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
}
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_valid_metrics_and_dimensions(
|
||||
cls,
|
||||
database: Database,
|
||||
table: SqlaTable,
|
||||
dimensions: set[str],
|
||||
metrics: set[str],
|
||||
) -> ValidColumnsType:
|
||||
"""
|
||||
Get valid metrics and dimensions.
|
||||
|
||||
Given a datasource, and sets of selected metrics and dimensions, return the
|
||||
sets of valid metrics and dimensions that can further be selected.
|
||||
"""
|
||||
if cls.semantic_layer:
|
||||
with database.get_sqla_engine() as engine:
|
||||
semantic_layer = cls.semantic_layer(engine)
|
||||
semantic_views = {
|
||||
semantic_view.name: semantic_view
|
||||
for semantic_view in semantic_layer.get_semantic_views()
|
||||
}
|
||||
if semantic_view := semantic_views.get(table.table):
|
||||
selected_metrics = {
|
||||
metric
|
||||
for metric in semantic_layer.get_metrics(semantic_view)
|
||||
if metric.name in metrics
|
||||
}
|
||||
selected_dimensions = {
|
||||
dimension
|
||||
for dimension in semantic_layer.get_dimensions(semantic_view)
|
||||
if dimension.name in dimensions
|
||||
}
|
||||
return {
|
||||
"metrics": {
|
||||
metric.name
|
||||
for metric in semantic_layer.get_valid_metrics(
|
||||
semantic_view,
|
||||
selected_metrics,
|
||||
selected_dimensions,
|
||||
)
|
||||
},
|
||||
"dimensions": {
|
||||
dimension.name
|
||||
for dimension in semantic_layer.get_valid_dimensions(
|
||||
semantic_view,
|
||||
selected_metrics,
|
||||
selected_dimensions,
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"dimensions": {column.column_name for column in table.columns},
|
||||
"metrics": {metric.metric_name for metric in table.metrics},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def where_latest_partition( # pylint: disable=unused-argument
|
||||
cls,
|
||||
@@ -1981,11 +1843,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||
:param kwargs: kwargs to be passed to cursor.execute()
|
||||
:return:
|
||||
"""
|
||||
if cls.semantic_layer:
|
||||
with cls.get_engine(database, schema="tpcds_sf10tcl") as engine:
|
||||
semantic_layer = cls.semantic_layer(engine)
|
||||
query = semantic_layer.get_query_from_standard_sql(query).sql
|
||||
|
||||
if cls.arraysize:
|
||||
cursor.arraysize = cls.arraysize
|
||||
try:
|
||||
|
||||
@@ -16,13 +16,11 @@
|
||||
# under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from re import Pattern
|
||||
from typing import Any, Iterator, Optional, TYPE_CHECKING, TypedDict
|
||||
from typing import Any, Optional, TYPE_CHECKING, TypedDict
|
||||
from urllib import parse
|
||||
|
||||
from apispec import APISpec
|
||||
@@ -32,48 +30,20 @@ from cryptography.hazmat.primitives import serialization
|
||||
from flask import current_app
|
||||
from flask_babel import gettext as __
|
||||
from marshmallow import fields, Schema
|
||||
from sqlalchemy import text, types
|
||||
from sqlalchemy.engine.interfaces import Dialect
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlglot import exp, parse_one
|
||||
|
||||
from superset.constants import TimeGrain
|
||||
from superset.databases.utils import make_url_safe
|
||||
from superset.db_engine_specs.base import BaseEngineSpec, BasicPropertiesType
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
|
||||
from superset.extensions.semantic_layer import (
|
||||
BINARY,
|
||||
BOOLEAN,
|
||||
Column as SemanticColumn,
|
||||
DATE,
|
||||
DATETIME,
|
||||
DECIMAL,
|
||||
Dimension as SemanticDimension,
|
||||
Filter as SemanticFilter,
|
||||
INTEGER,
|
||||
Metric as SemanticMetric,
|
||||
NoSort,
|
||||
NUMBER,
|
||||
OBJECT,
|
||||
Query as SemanticQuery,
|
||||
SemanticView,
|
||||
Sort as SemanticSort,
|
||||
SortDirectionEnum,
|
||||
STRING,
|
||||
Table as SemanticTable,
|
||||
TIME,
|
||||
Type as SemanticType,
|
||||
)
|
||||
from superset.models.sql_lab import Query
|
||||
from superset.sql.parse import Table
|
||||
from superset.utils import json
|
||||
from superset.utils.core import get_user_agent, QuerySource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.engine.base import Engine
|
||||
|
||||
from superset.models.core import Database
|
||||
|
||||
# Regular expressions to catch custom errors
|
||||
@@ -107,303 +77,6 @@ class SnowflakeParametersType(TypedDict):
|
||||
warehouse: str
|
||||
|
||||
|
||||
class SnowflakeSemanticLayer:
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
self.engine = engine
|
||||
|
||||
def execute(
|
||||
self,
|
||||
sql: str,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
with self.engine.connect() as connection:
|
||||
for row in connection.execute(text(sql), kwargs).mappings():
|
||||
yield dict(row)
|
||||
|
||||
def get_semantic_views(self) -> set[SemanticView]:
|
||||
sql = """
|
||||
SHOW SEMANTIC VIEWS
|
||||
->> SELECT "name" FROM $1;
|
||||
"""
|
||||
return {SemanticView(row["name"]) for row in self.execute(sql)}
|
||||
|
||||
def get_type(self, snowflake_type: str | None) -> type[SemanticType]:
|
||||
if snowflake_type is None:
|
||||
return STRING
|
||||
|
||||
type_map = {
|
||||
STRING: {r"VARCHAR\(\d+\)$", "STRING$", "TEXT$", r"CHAR\(\d+\)$"},
|
||||
INTEGER: {r"NUMBER\(38,\s?0\)$", "INT$", "INTEGER$", "BIGINT$"},
|
||||
DECIMAL: {r"NUMBER\(10,\s?2\)$"},
|
||||
NUMBER: {r"NUMBER\(\d+,\s?\d+\)$", "FLOAT$", "DOUBLE$"},
|
||||
BOOLEAN: {"BOOLEAN$"},
|
||||
DATE: {"DATE$"},
|
||||
DATETIME: {"TIMESTAMP_TZ$", "TIMESTAMP__NTZ$"},
|
||||
TIME: {"TIME$"},
|
||||
OBJECT: {"OBJECT$"},
|
||||
BINARY: {r"BINARY\(\d+\)$", r"VARBINARY\(\d+\)$"},
|
||||
}
|
||||
for semantic_type, patterns in type_map.items():
|
||||
if any(
|
||||
re.match(pattern, snowflake_type, re.IGNORECASE) for pattern in patterns
|
||||
):
|
||||
return semantic_type
|
||||
|
||||
return STRING
|
||||
|
||||
@classmethod
|
||||
def quote_table(cls, table: Table, dialect: Dialect) -> str:
|
||||
"""
|
||||
Fully quote a table name, including the schema and catalog.
|
||||
"""
|
||||
quoters = {
|
||||
"catalog": dialect.identifier_preparer.quote_schema,
|
||||
"schema": dialect.identifier_preparer.quote_schema,
|
||||
"table": dialect.identifier_preparer.quote,
|
||||
}
|
||||
|
||||
return ".".join(
|
||||
function(getattr(table, key))
|
||||
for key, function in quoters.items()
|
||||
if getattr(table, key)
|
||||
)
|
||||
|
||||
def get_metrics(self, semantic_view: SemanticView) -> set[SemanticMetric]:
|
||||
quoted_semantic_view_name = self.quote_table(
|
||||
Table(semantic_view.name),
|
||||
self.engine.dialect,
|
||||
)
|
||||
sql = f"""
|
||||
DESC SEMANTIC VIEW {quoted_semantic_view_name}
|
||||
->> SELECT "object_name", "property", "property_value"
|
||||
FROM $1
|
||||
WHERE
|
||||
"object_kind" = 'METRIC' AND
|
||||
"property" IN ('DATA_TYPE', 'TABLE');
|
||||
""" # noqa: S608 (semantic_view.name is quoted)
|
||||
rows = self.execute(sql)
|
||||
|
||||
metrics: set[SemanticMetric] = set()
|
||||
for name, group in itertools.groupby(rows, key=lambda x: x["object_name"]):
|
||||
attributes = defaultdict(set)
|
||||
for row in group:
|
||||
attributes[row["property"]].add(row["property_value"])
|
||||
|
||||
table = next(iter(attributes["TABLE"]))
|
||||
metric_name = table + "." + name
|
||||
type_ = self.get_type(next(iter(attributes["DATA_TYPE"])))
|
||||
sql = self.engine.dialect.identifier_preparer.quote(metric_name)
|
||||
tables = frozenset(attributes["TABLE"])
|
||||
join_columns = frozenset()
|
||||
|
||||
metrics.add(SemanticMetric(metric_name, type_, sql, tables, join_columns))
|
||||
|
||||
return metrics
|
||||
|
||||
def get_dimensions(self, semantic_view: SemanticView) -> set[SemanticDimension]:
|
||||
quoted_semantic_view_name = self.quote_table(
|
||||
Table(semantic_view.name),
|
||||
self.engine.dialect,
|
||||
)
|
||||
sql = f"""
|
||||
DESC SEMANTIC VIEW {quoted_semantic_view_name}
|
||||
->> SELECT "object_name", "property", "property_value"
|
||||
FROM $1
|
||||
WHERE
|
||||
"object_kind" = 'DIMENSION' AND
|
||||
"property" IN ('DATA_TYPE', 'TABLE');
|
||||
""" # noqa: S608 (semantic_view.name is quoted)
|
||||
rows = self.execute(sql)
|
||||
|
||||
dimensions: set[SemanticDimension] = set()
|
||||
for name, group in itertools.groupby(rows, key=lambda x: x["object_name"]):
|
||||
attributes = defaultdict(set)
|
||||
for row in group:
|
||||
attributes[row["property"]].add(row["property_value"])
|
||||
|
||||
table = next(iter(attributes["TABLE"]))
|
||||
dimension_name = table + "." + name
|
||||
column = SemanticColumn(SemanticTable(table), name)
|
||||
type_ = self.get_type(next(iter(attributes["DATA_TYPE"])))
|
||||
|
||||
dimensions.add(SemanticDimension(column, dimension_name, type_))
|
||||
|
||||
return dimensions
|
||||
|
||||
def get_valid_metrics(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[SemanticMetric],
|
||||
dimensions: set[SemanticDimension],
|
||||
) -> set[SemanticMetric]:
|
||||
# all metrics and dimensions are valid inside a given semantic view
|
||||
return self.get_metrics(semantic_view)
|
||||
|
||||
def get_valid_dimensions(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[SemanticMetric],
|
||||
dimensions: set[SemanticDimension],
|
||||
) -> set[SemanticDimension]:
|
||||
# all metrics and dimensions are valid inside a given semantic view
|
||||
return self.get_dimensions(semantic_view)
|
||||
|
||||
def get_query(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[SemanticMetric],
|
||||
dimensions: set[SemanticDimension],
|
||||
filters: set[SemanticFilter],
|
||||
sort: SemanticSort = NoSort,
|
||||
limit: int | None = None,
|
||||
offset: int | None = None,
|
||||
) -> SemanticQuery:
|
||||
ast = self.build_query(
|
||||
semantic_view,
|
||||
metrics,
|
||||
dimensions,
|
||||
filters,
|
||||
sort,
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
return SemanticQuery(sql=ast.sql(dialect="snowflake", pretty=True))
|
||||
|
||||
def build_query(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[SemanticMetric],
|
||||
dimensions: set[SemanticDimension],
|
||||
filters: set[SemanticFilter],
|
||||
sort: SemanticSort = NoSort,
|
||||
limit: int | None = None,
|
||||
offset: int | None = None,
|
||||
) -> exp.Select:
|
||||
semantic_view = exp.SemanticView(
|
||||
this=exp.Table(this=exp.Identifier(this=semantic_view.name, quoted=True)),
|
||||
dimensions=[
|
||||
exp.Column(
|
||||
this=exp.Identifier(this=dimension.column.name, quoted=True),
|
||||
table=exp.Identifier(
|
||||
this=dimension.column.relation.name,
|
||||
quoted=True,
|
||||
),
|
||||
)
|
||||
for dimension in dimensions
|
||||
],
|
||||
metrics=[
|
||||
exp.Column(
|
||||
this=exp.Identifier(this=column, quoted=True),
|
||||
table=exp.Identifier(this=table, quoted=True),
|
||||
)
|
||||
for table, column in (
|
||||
metric.name.split(".", 1)
|
||||
for metric in metrics
|
||||
if "." in metric.name
|
||||
)
|
||||
],
|
||||
)
|
||||
query = exp.Select(
|
||||
expressions=[exp.Star()],
|
||||
**{"from": exp.From(this=exp.Table(this=semantic_view))},
|
||||
)
|
||||
|
||||
if sort.items:
|
||||
order = [
|
||||
exp.Ordered(
|
||||
this=exp.Column(this=exp.Identifier(this=item.field.name)),
|
||||
desc=item.direction == SortDirectionEnum.DESC,
|
||||
nulls_first=item.nulls_first,
|
||||
)
|
||||
for item in sort.items
|
||||
]
|
||||
query.args["order"] = exp.Order(expressions=order)
|
||||
|
||||
if offset:
|
||||
query = query.offset(offset)
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
return query
|
||||
|
||||
def get_query_from_standard_sql(self, sql: str) -> SemanticQuery:
|
||||
"""
|
||||
Convert the Explore query into a proper query.
|
||||
|
||||
Explore will produce a pseudo-SQL query that references metrics and dimensions
|
||||
as if they were columns in a table. This method replaces the table name with a
|
||||
call to `SEMANTIC_VIEW`, and removes the `GROUP BY` clause, since all the
|
||||
aggregations happen inside the `SEMANTIC_VIEW` call.
|
||||
"""
|
||||
ast = parse_one(sql, "snowflake")
|
||||
table = ast.find(exp.Table)
|
||||
if not table:
|
||||
return SemanticQuery(sql=sql)
|
||||
|
||||
semantic_views = self.get_semantic_views()
|
||||
if table.name not in {semantic_view.name for semantic_view in semantic_views}:
|
||||
return SemanticQuery(sql=sql)
|
||||
|
||||
# collect all metric and dimensions
|
||||
semantic_view = SemanticView(table.name)
|
||||
all_metrics = self.get_metrics(semantic_view)
|
||||
all_dimensions = self.get_dimensions(semantic_view)
|
||||
|
||||
# collect metrics and dimensions used in the query
|
||||
columns = {column.name for column in ast.find_all(exp.Column)}
|
||||
metrics = [metric for metric in all_metrics if metric.name in columns]
|
||||
dimensions = [
|
||||
dimension for dimension in all_dimensions if dimension.name in columns
|
||||
]
|
||||
|
||||
# now replace table with a call to `SEMANTIC_VIEW`
|
||||
udtf = exp.Table(
|
||||
this=exp.SemanticView(
|
||||
this=exp.Table(
|
||||
this=exp.Identifier(this=semantic_view.name, quoted=True)
|
||||
),
|
||||
metrics=[
|
||||
exp.Column(
|
||||
this=exp.Identifier(this=column, quoted=True),
|
||||
table=exp.Identifier(this=table, quoted=True),
|
||||
)
|
||||
for table, column in (
|
||||
metric.name.split(".", 1)
|
||||
for metric in metrics
|
||||
if "." in metric.name
|
||||
)
|
||||
],
|
||||
dimensions=[
|
||||
exp.Column(
|
||||
this=exp.Identifier(this=column, quoted=True),
|
||||
table=exp.Identifier(this=table, quoted=True),
|
||||
)
|
||||
for table, column in (
|
||||
dimension.name.split(".", 1)
|
||||
for dimension in dimensions
|
||||
if "." in dimension.name
|
||||
)
|
||||
],
|
||||
),
|
||||
alias=exp.TableAlias(
|
||||
this=exp.Identifier(this="table_alias", quoted=False),
|
||||
columns=[
|
||||
exp.Identifier(this=column.name, quoted=True)
|
||||
for column in metrics + dimensions
|
||||
],
|
||||
),
|
||||
)
|
||||
table.replace(udtf)
|
||||
|
||||
# remove group by, since aggregations are done inside the `SEMANTIC_VIEW` call
|
||||
del ast.args["group"]
|
||||
|
||||
print("BETO")
|
||||
print(ast.sql(dialect="snowflake", pretty=True))
|
||||
return SemanticQuery(sql=ast.sql(dialect="snowflake", pretty=True))
|
||||
|
||||
|
||||
class SnowflakeEngineSpec(PostgresBaseEngineSpec):
|
||||
engine = "snowflake"
|
||||
engine_name = "Snowflake"
|
||||
@@ -417,8 +90,6 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
|
||||
default_driver = "snowflake"
|
||||
sqlalchemy_uri_placeholder = "snowflake://"
|
||||
|
||||
semantic_layer = SnowflakeSemanticLayer
|
||||
|
||||
supports_dynamic_schema = True
|
||||
supports_catalog = supports_dynamic_catalog = supports_cross_catalog_queries = True
|
||||
|
||||
|
||||
@@ -1,340 +0,0 @@
|
||||
import enum
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from functools import total_ordering
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from sqlalchemy import types as sqltypes
|
||||
from sqlalchemy.engine.base import Engine
|
||||
|
||||
|
||||
class Type:
|
||||
"""
|
||||
Base class for types.
|
||||
"""
|
||||
|
||||
|
||||
class INTEGER(Type):
|
||||
"""
|
||||
Represents an integer type.
|
||||
"""
|
||||
|
||||
|
||||
class NUMBER(Type):
|
||||
"""
|
||||
Represents a number type.
|
||||
"""
|
||||
|
||||
|
||||
class DECIMAL(Type):
|
||||
"""
|
||||
Represents a decimal type.
|
||||
"""
|
||||
|
||||
|
||||
class STRING(Type):
|
||||
"""
|
||||
Represents a string type.
|
||||
"""
|
||||
|
||||
|
||||
class BOOLEAN(Type):
|
||||
"""
|
||||
Represents a boolean type.
|
||||
"""
|
||||
|
||||
|
||||
class DATE(Type):
|
||||
"""
|
||||
Represents a date type.
|
||||
"""
|
||||
|
||||
|
||||
class TIME(Type):
|
||||
"""
|
||||
Represents a time type.
|
||||
"""
|
||||
|
||||
|
||||
class DATETIME(DATE, TIME):
|
||||
"""
|
||||
Represents a datetime type.
|
||||
"""
|
||||
|
||||
|
||||
class INTERVAL(Type):
|
||||
"""
|
||||
Represents an interval type.
|
||||
"""
|
||||
|
||||
|
||||
class OBJECT(Type):
|
||||
"""
|
||||
Represents an object type.
|
||||
"""
|
||||
|
||||
|
||||
class BINARY(Type):
|
||||
"""
|
||||
Represents a binary type.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticView:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Relation:
|
||||
name: str
|
||||
schema: str | None = None
|
||||
catalog: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Table:
|
||||
name: str
|
||||
schema: str | None = None
|
||||
catalog: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class View:
|
||||
name: str
|
||||
sql: str
|
||||
schema: str | None = None
|
||||
catalog: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Virtual:
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Metric:
|
||||
name: str
|
||||
type: type[Type]
|
||||
sql: str
|
||||
tables: frozenset[Table]
|
||||
join_columns: frozenset[str]
|
||||
|
||||
|
||||
@total_ordering
|
||||
class ComparableEnum(enum.Enum):
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if isinstance(other, enum.Enum):
|
||||
return self.value == other.value
|
||||
return NotImplemented
|
||||
|
||||
def __lt__(self, other: object) -> bool:
|
||||
if isinstance(other, enum.Enum):
|
||||
return self.value < other.value
|
||||
return NotImplemented
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.__class__, self.name))
|
||||
|
||||
|
||||
class TimeGrain(ComparableEnum):
|
||||
second = timedelta(seconds=1)
|
||||
minute = timedelta(minutes=1)
|
||||
hour = timedelta(hours=1)
|
||||
|
||||
|
||||
class DateGrain(ComparableEnum):
|
||||
day = timedelta(days=1)
|
||||
week = timedelta(weeks=1)
|
||||
month = timedelta(days=30)
|
||||
quarter = timedelta(days=90)
|
||||
year = timedelta(days=365)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Column:
|
||||
relation: Table | View | Virtual
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dimension:
|
||||
column: Column
|
||||
name: str
|
||||
type: type[Type]
|
||||
grain: TimeGrain | DateGrain | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
metadata = f"[{self.grain.name}]" if self.grain else ""
|
||||
return f"{self.type.__name__} {self.name} {metadata}".strip()
|
||||
|
||||
|
||||
class FilterTypeEnum(enum.Enum):
|
||||
WHERE = enum.auto()
|
||||
HAVING = enum.auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Filter:
|
||||
type: FilterTypeEnum
|
||||
expression: str
|
||||
|
||||
|
||||
class SortDirectionEnum(enum.Enum):
|
||||
ASC = enum.auto()
|
||||
DESC = enum.auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SortField:
|
||||
field: Metric | Dimension
|
||||
direction: SortDirectionEnum
|
||||
nulls_first: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sort:
|
||||
items: list[SortField]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Query:
|
||||
sql: str
|
||||
|
||||
|
||||
NoSort = Sort(items=[])
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SemanticLayer(Protocol):
|
||||
"""
|
||||
A generic protocol for semantic layers.
|
||||
"""
|
||||
|
||||
def __init__(self, engine: Engine) -> None: ...
|
||||
|
||||
def get_semantic_views(self) -> set[SemanticView]:
|
||||
"""
|
||||
Return a set of the semantic views.
|
||||
|
||||
A semantic view is an organizational group of metrics and dimensions. It's not a
|
||||
logical grouping, since metrics and dimensions from a given semantic view might
|
||||
not be compatible. An implementation might expose a single semantic view for
|
||||
exploration of available metric and dimesnions, and smaller curated semantic
|
||||
views that are domain specific.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_metrics(self, semantic_view: SemanticView) -> set[Metric]:
|
||||
"""
|
||||
Return a set of metrics from a given semantic views.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_dimensions(self, semantic_view: SemanticView) -> set[Dimension]:
|
||||
"""
|
||||
Return a set of dimensions from a given semantic views.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_valid_metrics(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[Metric],
|
||||
dimensions: set[Dimension],
|
||||
) -> set[Metric]:
|
||||
"""
|
||||
Return compatible metrics for the given metrics and dimensions.
|
||||
|
||||
For metrics to be valid they must be compatible with all the provided
|
||||
dimensions.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_valid_dimensions(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[Metric],
|
||||
dimensions: set[Dimension],
|
||||
) -> set[Dimension]:
|
||||
"""
|
||||
Return compatible dimensions for the given metrics.
|
||||
|
||||
For dimensions to be valid they must be compatible with all the provided
|
||||
metrics.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_query(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
metrics: set[Metric],
|
||||
dimensions: set[Dimension],
|
||||
# populations: set[Population],
|
||||
filters: set[Filter],
|
||||
sort: Sort = NoSort,
|
||||
limit: int | None = None,
|
||||
offset: int | None = None,
|
||||
) -> Query:
|
||||
"""
|
||||
Build a SQL query from the given metrics, dimensions, filters, and sort order.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_query_from_standard_sql(
|
||||
self,
|
||||
semantic_view: SemanticView,
|
||||
sql: str,
|
||||
) -> Query:
|
||||
"""
|
||||
Build a SQL query from a pseudo-query referencing metrics and dimensions.
|
||||
|
||||
For example, given `metric1` having the expression `COUNT(*)`, this query:
|
||||
|
||||
SELECT metric1, dim1
|
||||
FROM semantic_layer
|
||||
GROUP BY dim1
|
||||
|
||||
Becomes:
|
||||
|
||||
SELECT metric1, dim1
|
||||
FROM (
|
||||
SELECT COUNT(*) AS metric1, dim1
|
||||
FROM fact_table
|
||||
JOIN dim_table
|
||||
ON fact_table.dim_id = dim_table.id
|
||||
GROUP BY dim1
|
||||
) AS semantic_view
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
TYPE_MAPPING: dict[Type, type[sqltypes.TypeEngine]] = {
|
||||
# Numeric types
|
||||
INTEGER: sqltypes.Integer,
|
||||
NUMBER: sqltypes.Numeric,
|
||||
DECIMAL: sqltypes.DECIMAL,
|
||||
# String types
|
||||
STRING: sqltypes.String,
|
||||
# Boolean type
|
||||
BOOLEAN: sqltypes.Boolean,
|
||||
# Date/time types
|
||||
DATE: sqltypes.Date,
|
||||
TIME: sqltypes.Time,
|
||||
DATETIME: sqltypes.DateTime,
|
||||
INTERVAL: sqltypes.Interval,
|
||||
# Complex types
|
||||
OBJECT: sqltypes.JSON,
|
||||
BINARY: sqltypes.LargeBinary,
|
||||
}
|
||||
|
||||
|
||||
def get_sqla_type_from_dimension_type(
|
||||
dimension_type: Type,
|
||||
) -> sqltypes.TypeEngine:
|
||||
"""
|
||||
Get the SQLAlchemy type corresponding to the given dimension type.
|
||||
"""
|
||||
return TYPE_MAPPING.get(dimension_type, sqltypes.String)()
|
||||
@@ -126,9 +126,7 @@ class ConfigurationMethod(StrEnum):
|
||||
DYNAMIC_FORM = "dynamic_form"
|
||||
|
||||
|
||||
class Database(
|
||||
Model, AuditMixinNullable, ImportExportMixin
|
||||
): # pylint: disable=too-many-public-methods
|
||||
class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable=too-many-public-methods
|
||||
"""An ORM object that stores Database related information"""
|
||||
|
||||
__tablename__ = "dbs"
|
||||
@@ -402,7 +400,9 @@ class Database(
|
||||
return (
|
||||
username
|
||||
if (username := get_username())
|
||||
else object_url.username if self.impersonate_user else None
|
||||
else object_url.username
|
||||
if self.impersonate_user
|
||||
else None
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
@@ -987,10 +987,7 @@ class Database(
|
||||
schema=table.schema,
|
||||
) as inspector:
|
||||
return self.db_engine_spec.get_columns(
|
||||
self,
|
||||
inspector,
|
||||
table,
|
||||
self.schema_options,
|
||||
inspector, table, self.schema_options
|
||||
)
|
||||
|
||||
def get_metrics(
|
||||
@@ -1079,11 +1076,9 @@ class Database(
|
||||
return self.perm
|
||||
|
||||
def has_table(self, table: Table) -> bool:
|
||||
with self.get_inspector(
|
||||
catalog=table.catalog,
|
||||
schema=table.schema,
|
||||
) as inspector:
|
||||
return self.db_engine_spec.has_table(self, inspector, table)
|
||||
with self.get_sqla_engine(catalog=table.catalog, schema=table.schema) as engine:
|
||||
# do not pass "" as an empty schema; force null
|
||||
return engine.has_table(table.table, table.schema or None)
|
||||
|
||||
def has_view(self, table: Table) -> bool:
|
||||
with self.get_sqla_engine(catalog=table.catalog, schema=table.schema) as engine:
|
||||
|
||||
Reference in New Issue
Block a user