mirror of
https://github.com/apache/superset.git
synced 2026-06-14 12:09:14 +00:00
Compare commits
5 Commits
fix_sqllab
...
impala-dia
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5fb5dcd54 | ||
|
|
604d72cc98 | ||
|
|
913e068113 | ||
|
|
1a4e2173f5 | ||
|
|
c49789167b |
@@ -20,6 +20,9 @@
|
||||
# If you choose to use this type of deployment make sure to
|
||||
# create you own docker environment file (docker/.env) with your own
|
||||
# unique random secure passwords and SECRET_KEY.
|
||||
#
|
||||
# For verbose logging during development:
|
||||
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
|
||||
# -----------------------------------------------------------------------
|
||||
x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev}
|
||||
x-superset-volumes:
|
||||
|
||||
157
docker-compose-light.yml
Normal file
157
docker-compose-light.yml
Normal file
@@ -0,0 +1,157 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Lightweight docker-compose for running multiple Superset instances
|
||||
# This includes only essential services: database, Redis, and Superset app
|
||||
#
|
||||
# IMPORTANT: To run multiple instances in parallel:
|
||||
# - Use different project names: docker-compose -p project1 -f docker-compose-light.yml up
|
||||
# - Use different NODE_PORT values: NODE_PORT=9002 docker-compose -p project2 -f docker-compose-light.yml up
|
||||
# - Volumes are isolated by project name (e.g., project1_db_home_light, project2_db_home_light)
|
||||
# - Database name is intentionally different (superset_light) to prevent accidental cross-connections
|
||||
#
|
||||
# For verbose logging during development:
|
||||
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
|
||||
# -----------------------------------------------------------------------
|
||||
x-superset-user: &superset-user root
|
||||
x-superset-volumes: &superset-volumes
|
||||
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
||||
- ./docker:/app/docker
|
||||
- ./superset:/app/superset
|
||||
- ./superset-frontend:/app/superset-frontend
|
||||
- superset_home_light:/app/superset_home
|
||||
- ./tests:/app/tests
|
||||
x-common-build: &common-build
|
||||
context: .
|
||||
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
|
||||
cache_from:
|
||||
- apache/superset-cache:3.10-slim-bookworm
|
||||
args:
|
||||
DEV_MODE: "true"
|
||||
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
|
||||
INCLUDE_FIREFOX: ${INCLUDE_FIREFOX:-false}
|
||||
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
|
||||
|
||||
services:
|
||||
db-light:
|
||||
env_file:
|
||||
- path: docker/.env # default
|
||||
required: true
|
||||
- path: docker/.env-local # optional override
|
||||
required: false
|
||||
image: postgres:16
|
||||
restart: unless-stopped
|
||||
# No host port mapping - only accessible within Docker network
|
||||
volumes:
|
||||
- db_home_light:/var/lib/postgresql/data
|
||||
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
|
||||
environment:
|
||||
# Override database name to avoid conflicts
|
||||
POSTGRES_DB: superset_light
|
||||
|
||||
superset-light:
|
||||
env_file:
|
||||
- path: docker/.env # default
|
||||
required: true
|
||||
- path: docker/.env-local # optional override
|
||||
required: false
|
||||
build:
|
||||
<<: *common-build
|
||||
command: ["/app/docker/docker-bootstrap.sh", "app"]
|
||||
restart: unless-stopped
|
||||
# No host port mapping - accessed via webpack dev server proxy
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
user: *superset-user
|
||||
depends_on:
|
||||
superset-init-light:
|
||||
condition: service_completed_successfully
|
||||
volumes: *superset-volumes
|
||||
environment:
|
||||
# Override DB connection for light service
|
||||
DATABASE_HOST: db-light
|
||||
DATABASE_DB: superset_light
|
||||
POSTGRES_DB: superset_light
|
||||
EXAMPLES_HOST: db-light
|
||||
EXAMPLES_DB: superset_light
|
||||
EXAMPLES_USER: superset
|
||||
EXAMPLES_PASSWORD: superset
|
||||
# Use light-specific config that disables Redis
|
||||
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
|
||||
|
||||
superset-init-light:
|
||||
build:
|
||||
<<: *common-build
|
||||
command: ["/app/docker/docker-init.sh"]
|
||||
env_file:
|
||||
- path: docker/.env # default
|
||||
required: true
|
||||
- path: docker/.env-local # optional override
|
||||
required: false
|
||||
depends_on:
|
||||
db-light:
|
||||
condition: service_started
|
||||
user: *superset-user
|
||||
volumes: *superset-volumes
|
||||
environment:
|
||||
# Override DB connection for light service
|
||||
DATABASE_HOST: db-light
|
||||
DATABASE_DB: superset_light
|
||||
POSTGRES_DB: superset_light
|
||||
EXAMPLES_HOST: db-light
|
||||
EXAMPLES_DB: superset_light
|
||||
EXAMPLES_USER: superset
|
||||
EXAMPLES_PASSWORD: superset
|
||||
# Use light-specific config that disables Redis
|
||||
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
superset-node-light:
|
||||
build:
|
||||
context: .
|
||||
target: superset-node
|
||||
args:
|
||||
# This prevents building the frontend bundle since we'll mount local folder
|
||||
# and build it on startup while firing docker-frontend.sh in dev mode, where
|
||||
# it'll mount and watch local files and rebuild as you update them
|
||||
DEV_MODE: "true"
|
||||
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
|
||||
environment:
|
||||
# set this to false if you have perf issues running the npm i; npm run dev in-docker
|
||||
# if you do so, you have to run this manually on the host, which should perform better!
|
||||
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
|
||||
NPM_RUN_PRUNE: false
|
||||
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
|
||||
# configuring the dev-server to use the host.docker.internal to connect to the backend
|
||||
superset: "http://superset-light:8088"
|
||||
ports:
|
||||
- "127.0.0.1:${NODE_PORT:-9001}:9000" # Parameterized port
|
||||
command: ["/app/docker/docker-frontend.sh"]
|
||||
env_file:
|
||||
- path: docker/.env # default
|
||||
required: true
|
||||
- path: docker/.env-local # optional override
|
||||
required: false
|
||||
volumes: *superset-volumes
|
||||
|
||||
volumes:
|
||||
superset_home_light:
|
||||
external: false
|
||||
db_home_light:
|
||||
external: false
|
||||
@@ -20,6 +20,9 @@
|
||||
# If you choose to use this type of deployment make sure to
|
||||
# create you own docker environment file (docker/.env) with your own
|
||||
# unique random secure passwords and SECRET_KEY.
|
||||
#
|
||||
# For verbose logging during development:
|
||||
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
|
||||
# -----------------------------------------------------------------------
|
||||
x-superset-volumes:
|
||||
&superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
||||
|
||||
@@ -20,6 +20,9 @@
|
||||
# If you choose to use this type of deployment make sure to
|
||||
# create you own docker environment file (docker/.env) with your own
|
||||
# unique random secure passwords and SECRET_KEY.
|
||||
#
|
||||
# For verbose logging during development:
|
||||
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
|
||||
# -----------------------------------------------------------------------
|
||||
x-superset-user: &superset-user root
|
||||
x-superset-volumes: &superset-volumes
|
||||
|
||||
@@ -53,7 +53,12 @@ PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
|
||||
# Development and logging configuration
|
||||
# FLASK_DEBUG: Enables Flask dev features (auto-reload, better error pages) - keep 'true' for development
|
||||
FLASK_DEBUG=true
|
||||
# SUPERSET_LOG_LEVEL: Controls Superset application logging verbosity (debug, info, warning, error, critical)
|
||||
SUPERSET_LOG_LEVEL=info
|
||||
|
||||
SUPERSET_APP_ROOT="/"
|
||||
SUPERSET_ENV=development
|
||||
SUPERSET_LOAD_EXAMPLES=yes
|
||||
@@ -66,4 +71,3 @@ SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
|
||||
ENABLE_PLAYWRIGHT=false
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
|
||||
SUPERSET_LOG_LEVEL=info
|
||||
|
||||
1
docker/pythonpath_dev/.gitignore
vendored
1
docker/pythonpath_dev/.gitignore
vendored
@@ -20,4 +20,5 @@
|
||||
# DON'T ignore the .gitignore
|
||||
!.gitignore
|
||||
!superset_config.py
|
||||
!superset_config_docker_light.py
|
||||
!superset_config_local.example
|
||||
|
||||
@@ -129,7 +129,7 @@ if os.getenv("CYPRESS_CONFIG") == "true":
|
||||
#
|
||||
try:
|
||||
import superset_config_docker
|
||||
from superset_config_docker import * # noqa
|
||||
from superset_config_docker import * # noqa: F403
|
||||
|
||||
logger.info(
|
||||
f"Loaded your Docker configuration at [{superset_config_docker.__file__}]"
|
||||
|
||||
37
docker/pythonpath_dev/superset_config_docker_light.py
Normal file
37
docker/pythonpath_dev/superset_config_docker_light.py
Normal file
@@ -0,0 +1,37 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Configuration for docker-compose-light.yml - disables Redis and uses minimal services
|
||||
|
||||
# Import all settings from the main config first
|
||||
from flask_caching.backends.filesystemcache import FileSystemCache
|
||||
from superset_config import * # noqa: F403
|
||||
|
||||
# Override caching to use simple in-memory cache instead of Redis
|
||||
RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
|
||||
|
||||
CACHE_CONFIG = {
|
||||
"CACHE_TYPE": "SimpleCache",
|
||||
"CACHE_DEFAULT_TIMEOUT": 300,
|
||||
"CACHE_KEY_PREFIX": "superset_light_",
|
||||
}
|
||||
DATA_CACHE_CONFIG = CACHE_CONFIG
|
||||
THUMBNAIL_CACHE_CONFIG = CACHE_CONFIG
|
||||
|
||||
|
||||
# Disable Celery entirely for lightweight mode
|
||||
CELERY_CONFIG = None # type: ignore[assignment,misc]
|
||||
@@ -26,11 +26,14 @@ Superset locally is using Docker Compose on a Linux or Mac OSX
|
||||
computer. Superset does not have official support for Windows. It's also the easiest
|
||||
way to launch a fully functioning **development environment** quickly.
|
||||
|
||||
Note that there are 3 major ways we support to run `docker compose`:
|
||||
Note that there are 4 major ways we support to run `docker compose`:
|
||||
|
||||
1. **docker-compose.yml:** for interactive development, where we mount your local folder with the
|
||||
frontend/backend files that you can edit and experience the changes you
|
||||
make in the app in real time
|
||||
1. **docker-compose-light.yml:** a lightweight configuration with minimal services (database,
|
||||
Superset app, and frontend dev server) for development. Uses in-memory caching instead of Redis
|
||||
and is designed for running multiple instances simultaneously
|
||||
1. **docker-compose-non-dev.yml** where we just build a more immutable image based on the
|
||||
local branch and get all the required images running. Changes in the local branch
|
||||
at the time you fire this up will be reflected, but changes to the code
|
||||
@@ -44,7 +47,7 @@ Note that there are 3 major ways we support to run `docker compose`:
|
||||
The `dev` builds include the `psycopg2-binary` required to connect
|
||||
to the Postgres database launched as part of the `docker compose` builds.
|
||||
|
||||
More on these two approaches after setting up the requirements for either.
|
||||
More on these approaches after setting up the requirements for either.
|
||||
|
||||
## Requirements
|
||||
|
||||
@@ -103,13 +106,36 @@ and help you start fresh. In the context of `docker compose` setting
|
||||
from within docker. This will slow down the startup, but will fix various npm-related issues.
|
||||
:::
|
||||
|
||||
### Option #2 - build a set of immutable images from the local branch
|
||||
### Option #2 - lightweight development with multiple instances
|
||||
|
||||
For a lighter development setup that uses fewer resources and supports running multiple instances:
|
||||
|
||||
```bash
|
||||
# Single lightweight instance (default port 9001)
|
||||
docker compose -f docker-compose-light.yml up
|
||||
|
||||
# Multiple instances with different ports
|
||||
NODE_PORT=9001 docker compose -p superset-1 -f docker-compose-light.yml up
|
||||
NODE_PORT=9002 docker compose -p superset-2 -f docker-compose-light.yml up
|
||||
NODE_PORT=9003 docker compose -p superset-3 -f docker-compose-light.yml up
|
||||
```
|
||||
|
||||
This configuration includes:
|
||||
- PostgreSQL database (internal network only)
|
||||
- Superset application server
|
||||
- Frontend development server with webpack hot reloading
|
||||
- In-memory caching (no Redis)
|
||||
- Isolated volumes and networks per instance
|
||||
|
||||
Access each instance at `http://localhost:{NODE_PORT}` (e.g., `http://localhost:9001`).
|
||||
|
||||
### Option #3 - build a set of immutable images from the local branch
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose-non-dev.yml up
|
||||
```
|
||||
|
||||
### Option #3 - boot up an official release
|
||||
### Option #4 - boot up an official release
|
||||
|
||||
```bash
|
||||
# Set the version you want to run
|
||||
|
||||
@@ -149,7 +149,12 @@ export default styled.div`
|
||||
.dt-pagination {
|
||||
text-align: right;
|
||||
/* use padding instead of margin so clientHeight can capture it */
|
||||
padding-top: 0.5em;
|
||||
padding: ${theme.paddingXXS}px 0px;
|
||||
}
|
||||
|
||||
.dt-pagination .pagination > li {
|
||||
display: inline;
|
||||
margin: 0 ${theme.marginXXS}px;
|
||||
}
|
||||
|
||||
.dt-pagination .pagination > li > a,
|
||||
@@ -157,6 +162,8 @@ export default styled.div`
|
||||
background-color: ${theme.colorBgBase};
|
||||
color: ${theme.colorText};
|
||||
border-color: ${theme.colorBorderSecondary};
|
||||
padding: ${theme.paddingXXS}px ${theme.paddingXS}px;
|
||||
border-radius: ${theme.borderRadius}px;
|
||||
}
|
||||
|
||||
.dt-pagination .pagination > li.active > a,
|
||||
|
||||
@@ -96,7 +96,6 @@ const StyledTabsContainer = styled.div`
|
||||
|
||||
.ant-tabs-content-holder {
|
||||
overflow: visible;
|
||||
padding-top: ${theme.sizeUnit * 4}px;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
import { memo, useMemo } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
import { css, SupersetTheme } from '@superset-ui/core';
|
||||
import { Icons } from '@superset-ui/core/components/Icons';
|
||||
import { Flex, Icons } from '@superset-ui/core/components';
|
||||
import { getChartKey } from 'src/explore/exploreUtils';
|
||||
import { ExplorePageState } from 'src/explore/types';
|
||||
import { FastVizSwitcherProps } from './types';
|
||||
@@ -79,14 +79,7 @@ export const FastVizSwitcher = memo(
|
||||
}, [currentSelection, currentViz]);
|
||||
|
||||
return (
|
||||
<div
|
||||
css={(theme: SupersetTheme) => css`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
column-gap: ${theme.sizeUnit}px;
|
||||
`}
|
||||
data-test="fast-viz-switcher"
|
||||
>
|
||||
<Flex justify="space-between" gap={4} data-test="fast-viz-switcher">
|
||||
{vizTiles.map(vizMeta => (
|
||||
<VizTile
|
||||
vizMeta={vizMeta}
|
||||
@@ -96,7 +89,7 @@ export const FastVizSwitcher = memo(
|
||||
key={vizMeta.name}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</Flex>
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -124,7 +124,7 @@ export const VizTile = ({
|
||||
>
|
||||
<span
|
||||
css={css`
|
||||
padding: 0px ${theme.sizeUnit}px;
|
||||
padding: 0px ${theme.sizeUnit * 1.25}px;
|
||||
`}
|
||||
>
|
||||
{vizMeta.icon}
|
||||
@@ -136,6 +136,7 @@ export const VizTile = ({
|
||||
font-size: ${theme.fontSizeSM}px;
|
||||
min-width: 0;
|
||||
padding-right: ${theme.sizeUnit}px;
|
||||
line-height: 1;
|
||||
`}
|
||||
ref={chartNameRef}
|
||||
>
|
||||
|
||||
@@ -612,3 +612,42 @@ test('should render an extension component if one is supplied', async () => {
|
||||
|
||||
expect(extension[0]).toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('should render the brand text if available', async () => {
|
||||
useSelectorMock.mockReturnValue({ roles: [] });
|
||||
|
||||
const modifiedProps = {
|
||||
...mockedProps,
|
||||
data: {
|
||||
...mockedProps.data,
|
||||
brand: {
|
||||
...mockedProps.data.brand,
|
||||
text: 'Welcome to Superset',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
render(<Menu {...modifiedProps} />, {
|
||||
useRouter: true,
|
||||
useQueryParams: true,
|
||||
useRedux: true,
|
||||
useTheme: true,
|
||||
});
|
||||
|
||||
const brandText = await screen.findByText('Welcome to Superset');
|
||||
expect(brandText).toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('should not render the brand text if not available', async () => {
|
||||
useSelectorMock.mockReturnValue({ roles: [] });
|
||||
const text = 'Welcome to Superset';
|
||||
render(<Menu {...mockedProps} />, {
|
||||
useRouter: true,
|
||||
useQueryParams: true,
|
||||
useRedux: true,
|
||||
useTheme: true,
|
||||
});
|
||||
|
||||
const brandText = screen.queryByText(text);
|
||||
expect(brandText).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
@@ -52,6 +52,8 @@ const StyledHeader = styled.header`
|
||||
display: none;
|
||||
}
|
||||
& .ant-image{
|
||||
display: contents;
|
||||
height: 100%;
|
||||
padding: ${theme.sizeUnit}px
|
||||
${theme.sizeUnit * 2}px
|
||||
${theme.sizeUnit}px
|
||||
@@ -87,7 +89,7 @@ const StyledHeader = styled.header`
|
||||
padding-left: ${theme.sizeUnit * 4}px;
|
||||
padding-right: ${theme.sizeUnit * 4}px;
|
||||
margin-right: ${theme.sizeUnit * 6}px;
|
||||
font-size: ${theme.sizeUnit * 4}px;
|
||||
font-size: ${theme.fontSizeLG}px;
|
||||
float: left;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -322,6 +324,11 @@ export function Menu({
|
||||
>
|
||||
{renderBrand()}
|
||||
</Tooltip>
|
||||
{brand.text && (
|
||||
<div className="navbar-brand-text">
|
||||
<span>{brand.text}</span>
|
||||
</div>
|
||||
)}
|
||||
<MainNav
|
||||
mode={showMenu}
|
||||
data-test="navbar-top"
|
||||
|
||||
@@ -53,6 +53,7 @@ const {
|
||||
measure = false,
|
||||
nameChunks = false,
|
||||
} = parsedArgs;
|
||||
|
||||
const isDevMode = mode !== 'production';
|
||||
const isDevServer = process.argv[1].includes('webpack-dev-server');
|
||||
|
||||
@@ -535,6 +536,11 @@ if (isDevMode) {
|
||||
runtimeErrors: error => !/ResizeObserver/.test(error.message),
|
||||
},
|
||||
logging: 'error',
|
||||
webSocketURL: {
|
||||
hostname: '0.0.0.0',
|
||||
pathname: '/ws',
|
||||
port: 0,
|
||||
},
|
||||
},
|
||||
static: {
|
||||
directory: path.join(process.cwd(), '../static/assets'),
|
||||
|
||||
@@ -1155,7 +1155,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods
|
||||
}
|
||||
|
||||
|
||||
CELERY_CONFIG: type[CeleryConfig] = CeleryConfig
|
||||
CELERY_CONFIG: type[CeleryConfig] | None = CeleryConfig
|
||||
|
||||
# Set celery config to None to disable all the above configuration
|
||||
# CELERY_CONFIG = None
|
||||
|
||||
@@ -17,5 +17,6 @@
|
||||
|
||||
from .dremio import Dremio
|
||||
from .firebolt import Firebolt, FireboltOld
|
||||
from .impala import Impala
|
||||
|
||||
__all__ = ["Dremio", "Firebolt", "FireboltOld"]
|
||||
__all__ = ["Dremio", "Firebolt", "FireboltOld", "Impala"]
|
||||
|
||||
259
superset/sql/dialects/impala.py
Normal file
259
superset/sql/dialects/impala.py
Normal file
@@ -0,0 +1,259 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing as t
|
||||
|
||||
from sqlglot import exp, generator, parser
|
||||
from sqlglot.dialects.hive import Hive
|
||||
from sqlglot.helper import seq_get
|
||||
|
||||
|
||||
class Impala(Hive):
|
||||
"""
|
||||
A sqlglot dialect for Impala.
|
||||
|
||||
Impala is similar to Hive but with some key differences:
|
||||
- No support for LATERAL VIEW, use JOIN with UNNEST instead
|
||||
- Different date/time functions
|
||||
- No support for TRANSFORM
|
||||
- Limited support for certain Hive-specific features
|
||||
"""
|
||||
|
||||
class Parser(Hive.Parser):
|
||||
FUNCTIONS = {
|
||||
**Hive.Parser.FUNCTIONS,
|
||||
# Impala-specific functions
|
||||
"MONTHS_ADD": lambda args: exp.DateAdd(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("MONTH"),
|
||||
),
|
||||
"MONTHS_SUB": lambda args: exp.DateSub(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("MONTH"),
|
||||
),
|
||||
"YEARS_ADD": lambda args: exp.DateAdd(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("YEAR"),
|
||||
),
|
||||
"YEARS_SUB": lambda args: exp.DateSub(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("YEAR"),
|
||||
),
|
||||
"DAYS_ADD": lambda args: exp.DateAdd(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("DAY"),
|
||||
),
|
||||
"DAYS_SUB": lambda args: exp.DateSub(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("DAY"),
|
||||
),
|
||||
"WEEKS_ADD": lambda args: exp.DateAdd(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("WEEK"),
|
||||
),
|
||||
"WEEKS_SUB": lambda args: exp.DateSub(
|
||||
this=seq_get(args, 0),
|
||||
expression=seq_get(args, 1),
|
||||
unit=exp.Literal.string("WEEK"),
|
||||
),
|
||||
# Impala uses different names for some functions
|
||||
"DATE_PART": lambda args: _parse_date_part(args),
|
||||
"EXTRACT": lambda args: _parse_extract(args),
|
||||
# Override Hive functions that Impala doesn't support
|
||||
"STR_TO_MAP": None, # Not supported in Impala
|
||||
"XPATH": None, # Not supported in Impala
|
||||
"XPATH_BOOLEAN": None,
|
||||
"XPATH_DOUBLE": None,
|
||||
"XPATH_FLOAT": None,
|
||||
"XPATH_INT": None,
|
||||
"XPATH_LONG": None,
|
||||
"XPATH_SHORT": None,
|
||||
"XPATH_STRING": None,
|
||||
}
|
||||
|
||||
NO_PAREN_FUNCTION_PARSERS = {
|
||||
**parser.Parser.NO_PAREN_FUNCTION_PARSERS,
|
||||
# Remove TRANSFORM as it's not supported in Impala
|
||||
}
|
||||
NO_PAREN_FUNCTION_PARSERS.pop("TRANSFORM", None)
|
||||
|
||||
def _parse_lateral(self) -> t.Optional[exp.Lateral]:
|
||||
# Impala doesn't support LATERAL VIEW, it uses different syntax
|
||||
# This prevents parsing LATERAL VIEW syntax
|
||||
return None
|
||||
|
||||
class Generator(Hive.Generator):
|
||||
# Impala-specific type mappings
|
||||
TYPE_MAPPING = {
|
||||
**Hive.Generator.TYPE_MAPPING,
|
||||
exp.DataType.Type.VARCHAR: "STRING", # Impala treats VARCHAR as STRING
|
||||
exp.DataType.Type.NVARCHAR: "STRING",
|
||||
exp.DataType.Type.CHAR: "STRING", # Impala treats CHAR as STRING
|
||||
exp.DataType.Type.NCHAR: "STRING",
|
||||
}
|
||||
|
||||
TRANSFORMS = {
|
||||
**Hive.Generator.TRANSFORMS,
|
||||
# Date/time functions
|
||||
exp.DateAdd: lambda self, e: _date_add_sql(self, e),
|
||||
exp.DateSub: lambda self, e: _date_sub_sql(self, e),
|
||||
# Impala doesn't support certain Hive features
|
||||
exp.StrToMap: lambda self, e: self.unsupported(
|
||||
"STR_TO_MAP is not supported in Impala"
|
||||
),
|
||||
exp.Transform: lambda self, e: self.unsupported(
|
||||
"TRANSFORM is not supported in Impala"
|
||||
),
|
||||
exp.QueryTransform: lambda self, e: self.unsupported(
|
||||
"TRANSFORM is not supported in Impala"
|
||||
),
|
||||
# Override LATERAL VIEW handling
|
||||
exp.Lateral: lambda self, e: _lateral_sql(self, e),
|
||||
# JSON functions have different names in Impala
|
||||
exp.JSONExtract: lambda self, e: self.func(
|
||||
"JSON_QUERY", e.this, e.expression
|
||||
),
|
||||
exp.JSONExtractScalar: lambda self, e: self.func(
|
||||
"JSON_VALUE", e.this, e.expression
|
||||
),
|
||||
# Impala uses different syntax for COLLECT_LIST/SET
|
||||
exp.ArrayAgg: lambda self, e: self.func(
|
||||
"GROUP_CONCAT",
|
||||
e.this.this if isinstance(e.this, exp.Order) else e.this,
|
||||
exp.Literal.string(","),
|
||||
),
|
||||
exp.ArrayUniqueAgg: lambda self, e: self.func(
|
||||
"GROUP_CONCAT",
|
||||
self.sql(exp.Distinct(expressions=[e.this])),
|
||||
exp.Literal.string(","),
|
||||
),
|
||||
}
|
||||
|
||||
def datatype_sql(self, expression: exp.DataType) -> str:
|
||||
# Impala treats CHAR/VARCHAR as STRING
|
||||
if expression.is_type("char", "varchar", "nchar", "nvarchar"):
|
||||
return "STRING"
|
||||
|
||||
return super().datatype_sql(expression)
|
||||
|
||||
def lateral_sql(self, expression: exp.Lateral) -> str:
|
||||
# Impala doesn't use LATERAL VIEW syntax
|
||||
# Instead, it uses regular JOIN with UNNEST
|
||||
if isinstance(expression.this, exp.Unnest):
|
||||
return self.sql(expression.this)
|
||||
return super().lateral_sql(expression)
|
||||
|
||||
|
||||
def _parse_date_part(args: t.List[exp.Expression]) -> exp.Expression:
|
||||
"""Parse DATE_PART function which extracts date parts."""
|
||||
part = seq_get(args, 0)
|
||||
date = seq_get(args, 1)
|
||||
|
||||
if isinstance(part, exp.Literal):
|
||||
part_name = part.name.upper()
|
||||
if part_name == "YEAR":
|
||||
return exp.Year(this=date)
|
||||
elif part_name == "MONTH":
|
||||
return exp.Month(this=date)
|
||||
elif part_name == "DAY":
|
||||
return exp.Day(this=date)
|
||||
elif part_name == "HOUR":
|
||||
return exp.Hour(this=date)
|
||||
elif part_name == "MINUTE":
|
||||
return exp.Minute(this=date)
|
||||
elif part_name == "SECOND":
|
||||
return exp.Second(this=date)
|
||||
|
||||
return exp.Extract(this=part, expression=date)
|
||||
|
||||
|
||||
def _parse_extract(args: t.List[exp.Expression]) -> exp.Expression:
|
||||
"""Parse EXTRACT function."""
|
||||
return exp.Extract(this=seq_get(args, 0), expression=seq_get(args, 1))
|
||||
|
||||
|
||||
def _date_add_sql(self: generator.Generator, expression: exp.DateAdd) -> str:
|
||||
"""Generate SQL for date addition in Impala."""
|
||||
unit = expression.text("unit").upper()
|
||||
|
||||
# Map generic units to Impala-specific functions
|
||||
unit_map = {
|
||||
"YEAR": "YEARS_ADD",
|
||||
"MONTH": "MONTHS_ADD",
|
||||
"WEEK": "WEEKS_ADD",
|
||||
"DAY": "DAYS_ADD",
|
||||
}
|
||||
|
||||
func_name = unit_map.get(unit, "DATE_ADD")
|
||||
|
||||
if func_name != "DATE_ADD":
|
||||
return self.func(func_name, expression.this, expression.expression)
|
||||
|
||||
# For other units, use DATE_ADD with INTERVAL
|
||||
return self.func(
|
||||
"DATE_ADD",
|
||||
expression.this,
|
||||
self.sql(exp.Interval(this=expression.expression, unit=expression.unit)),
|
||||
)
|
||||
|
||||
|
||||
def _date_sub_sql(self: generator.Generator, expression: exp.DateSub) -> str:
|
||||
"""Generate SQL for date subtraction in Impala."""
|
||||
unit = expression.text("unit").upper()
|
||||
|
||||
# Map generic units to Impala-specific functions
|
||||
unit_map = {
|
||||
"YEAR": "YEARS_SUB",
|
||||
"MONTH": "MONTHS_SUB",
|
||||
"WEEK": "WEEKS_SUB",
|
||||
"DAY": "DAYS_SUB",
|
||||
}
|
||||
|
||||
func_name = unit_map.get(unit, "DATE_SUB")
|
||||
|
||||
if func_name != "DATE_SUB":
|
||||
return self.func(func_name, expression.this, expression.expression)
|
||||
|
||||
# For other units, use DATE_SUB with INTERVAL
|
||||
return self.func(
|
||||
"DATE_SUB",
|
||||
expression.this,
|
||||
self.sql(exp.Interval(this=expression.expression, unit=expression.unit)),
|
||||
)
|
||||
|
||||
|
||||
def _lateral_sql(self: generator.Generator, expression: exp.Lateral) -> str:
|
||||
"""Generate SQL for LATERAL expressions in Impala."""
|
||||
# Impala doesn't support LATERAL VIEW syntax
|
||||
# It uses regular JOIN with UNNEST instead
|
||||
this = expression.this
|
||||
|
||||
if isinstance(this, exp.Unnest):
|
||||
# Just return the UNNEST expression without LATERAL VIEW
|
||||
return self.sql(this)
|
||||
|
||||
# For other cases, try to generate standard SQL
|
||||
return self.sql(this)
|
||||
@@ -44,7 +44,7 @@ from sqlglot.optimizer.scope import (
|
||||
)
|
||||
|
||||
from superset.exceptions import QueryClauseValidationException, SupersetParseError
|
||||
from superset.sql.dialects import Dremio, Firebolt
|
||||
from superset.sql.dialects import Dremio, Firebolt, Impala
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.models.core import Database
|
||||
@@ -81,7 +81,7 @@ SQLGLOT_DIALECTS = {
|
||||
"hana": Dialects.POSTGRES,
|
||||
"hive": Dialects.HIVE,
|
||||
# "ibmi": ???
|
||||
# "impala": ???
|
||||
"impala": Impala,
|
||||
# "kustosql": ???
|
||||
# "kylin": ???
|
||||
"mariadb": Dialects.MYSQL,
|
||||
|
||||
319
tests/unit_tests/sql/dialects/impala_tests.py
Normal file
319
tests/unit_tests/sql/dialects/impala_tests.py
Normal file
@@ -0,0 +1,319 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pytest
|
||||
from sqlglot import exp, parse_one
|
||||
|
||||
from superset.sql.dialects.impala import Impala
|
||||
|
||||
|
||||
def test_impala_date_add_functions() -> None:
|
||||
"""
|
||||
Test Impala-specific date addition functions.
|
||||
"""
|
||||
# Test MONTHS_ADD
|
||||
sql = "SELECT MONTHS_ADD(date_col, 3) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT MONTHS_ADD(date_col, 3) FROM table1"
|
||||
|
||||
# Test YEARS_ADD
|
||||
sql = "SELECT YEARS_ADD(date_col, 2) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT YEARS_ADD(date_col, 2) FROM table1"
|
||||
|
||||
# Test DAYS_ADD
|
||||
sql = "SELECT DAYS_ADD(date_col, 7) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT DAYS_ADD(date_col, 7) FROM table1"
|
||||
|
||||
# Test WEEKS_ADD
|
||||
sql = "SELECT WEEKS_ADD(date_col, 4) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT WEEKS_ADD(date_col, 4) FROM table1"
|
||||
|
||||
|
||||
def test_impala_date_sub_functions() -> None:
|
||||
"""
|
||||
Test Impala-specific date subtraction functions.
|
||||
"""
|
||||
# Test MONTHS_SUB
|
||||
sql = "SELECT MONTHS_SUB(date_col, 3) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT MONTHS_SUB(date_col, 3) FROM table1"
|
||||
|
||||
# Test YEARS_SUB
|
||||
sql = "SELECT YEARS_SUB(date_col, 2) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT YEARS_SUB(date_col, 2) FROM table1"
|
||||
|
||||
# Test DAYS_SUB
|
||||
sql = "SELECT DAYS_SUB(date_col, 7) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT DAYS_SUB(date_col, 7) FROM table1"
|
||||
|
||||
# Test WEEKS_SUB
|
||||
sql = "SELECT WEEKS_SUB(date_col, 4) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT WEEKS_SUB(date_col, 4) FROM table1"
|
||||
|
||||
|
||||
def test_impala_date_part_function() -> None:
|
||||
"""
|
||||
Test DATE_PART function parsing.
|
||||
"""
|
||||
# Test DATE_PART with YEAR
|
||||
sql = "SELECT DATE_PART('YEAR', date_col) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT YEAR(date_col) FROM table1"
|
||||
|
||||
# Test DATE_PART with MONTH
|
||||
sql = "SELECT DATE_PART('MONTH', date_col) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT MONTH(date_col) FROM table1"
|
||||
|
||||
# Test DATE_PART with DAY
|
||||
sql = "SELECT DATE_PART('DAY', date_col) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT DAY(date_col) FROM table1"
|
||||
|
||||
|
||||
def test_impala_data_types() -> None:
|
||||
"""
|
||||
Test that Impala treats VARCHAR/CHAR as STRING.
|
||||
"""
|
||||
# Test VARCHAR conversion
|
||||
sql = "CREATE TABLE test (col1 VARCHAR(100))"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "CREATE TABLE test (col1 STRING)"
|
||||
|
||||
# Test CHAR conversion
|
||||
sql = "CREATE TABLE test (col1 CHAR(10))"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "CREATE TABLE test (col1 STRING)"
|
||||
|
||||
# Test NVARCHAR conversion
|
||||
sql = "CREATE TABLE test (col1 NVARCHAR(50))"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "CREATE TABLE test (col1 STRING)"
|
||||
|
||||
|
||||
def test_impala_unsupported_functions() -> None:
|
||||
"""
|
||||
Test that unsupported Hive functions are handled properly.
|
||||
"""
|
||||
# STR_TO_MAP is not supported
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
sql = "SELECT STR_TO_MAP('a:1,b:2', ',', ':') FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
ast.sql(dialect=Impala)
|
||||
assert "STR_TO_MAP is not supported" in str(exc_info.value)
|
||||
|
||||
# TRANSFORM is not supported
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
sql = "SELECT TRANSFORM(col) USING 'script.py' FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
ast.sql(dialect=Impala)
|
||||
assert "TRANSFORM is not supported" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_impala_json_functions() -> None:
|
||||
"""
|
||||
Test that JSON functions are mapped to Impala equivalents.
|
||||
"""
|
||||
# Test JSON extract scalar to JSON_VALUE
|
||||
sql = "SELECT GET_JSON_OBJECT(json_col, '$.field') FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT JSON_VALUE(json_col, '$.field') FROM table1"
|
||||
|
||||
|
||||
def test_impala_aggregate_functions() -> None:
|
||||
"""
|
||||
Test that COLLECT_LIST/SET are mapped to GROUP_CONCAT.
|
||||
"""
|
||||
# Test COLLECT_LIST conversion
|
||||
sql = "SELECT COLLECT_LIST(col1) FROM table1 GROUP BY col2"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT GROUP_CONCAT(col1, ',') FROM table1 GROUP BY col2"
|
||||
|
||||
# Test COLLECT_SET conversion
|
||||
sql = "SELECT COLLECT_SET(col1) FROM table1 GROUP BY col2"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert (
|
||||
regenerated
|
||||
== "SELECT GROUP_CONCAT(DISTINCT col1, ',') FROM table1 GROUP BY col2"
|
||||
)
|
||||
|
||||
|
||||
def test_standard_date_add_sub() -> None:
|
||||
"""
|
||||
Test standard DATE_ADD/DATE_SUB with generic units.
|
||||
"""
|
||||
# Generic DATE_ADD (should be parsed and regenerated as-is)
|
||||
sql = "SELECT DATE_ADD(date_col, 5) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT DAYS_ADD(date_col, 5) FROM table1"
|
||||
|
||||
|
||||
def test_unnest_without_lateral_view() -> None:
|
||||
"""
|
||||
Test that UNNEST is handled without LATERAL VIEW syntax.
|
||||
"""
|
||||
# Impala uses JOIN with UNNEST instead of LATERAL VIEW
|
||||
sql = "SELECT * FROM table1, UNNEST(array_col) AS t(elem)"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
# Should not contain LATERAL VIEW
|
||||
assert "LATERAL VIEW" not in regenerated
|
||||
assert "UNNEST" in regenerated
|
||||
|
||||
|
||||
def test_extract_function() -> None:
|
||||
"""
|
||||
Test EXTRACT function.
|
||||
"""
|
||||
sql = "SELECT EXTRACT(YEAR FROM date_col) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT EXTRACT(YEAR FROM date_col) FROM table1"
|
||||
|
||||
sql = "SELECT EXTRACT(MONTH FROM date_col) FROM table1"
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
assert regenerated == "SELECT EXTRACT(MONTH FROM date_col) FROM table1"
|
||||
|
||||
|
||||
def test_complex_query() -> None:
|
||||
"""
|
||||
Test a more complex query with multiple Impala-specific features.
|
||||
"""
|
||||
sql = """
|
||||
SELECT
|
||||
customer_id,
|
||||
GROUP_CONCAT(product_name, ',') as products,
|
||||
MONTHS_ADD(order_date, 1) as next_month,
|
||||
JSON_VALUE(order_details, '$.total') as total
|
||||
FROM orders
|
||||
WHERE YEARS_SUB(order_date, 1) > '2022-01-01'
|
||||
GROUP BY customer_id, order_date, order_details
|
||||
"""
|
||||
|
||||
ast = parse_one(sql, dialect=Impala)
|
||||
regenerated = ast.sql(dialect=Impala)
|
||||
|
||||
# Check key components are preserved
|
||||
assert "GROUP_CONCAT" in regenerated
|
||||
assert "MONTHS_ADD" in regenerated
|
||||
assert "JSON_VALUE" in regenerated
|
||||
assert "YEARS_SUB" in regenerated
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"func, expected_class, unit",
|
||||
[
|
||||
("MONTHS_ADD(x, 1)", exp.DateAdd, "MONTH"),
|
||||
("YEARS_SUB(x, 2)", exp.DateSub, "YEAR"),
|
||||
("DAYS_ADD(x, 3)", exp.DateAdd, "DAY"),
|
||||
("WEEKS_SUB(x, 4)", exp.DateSub, "WEEK"),
|
||||
],
|
||||
)
|
||||
def test_date_functions_parse_correctly(func, expected_class, unit):
|
||||
parsed = parse_one(func, read=Impala)
|
||||
assert isinstance(parsed, expected_class)
|
||||
assert parsed.text("unit").upper() == unit
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sql, expected_expr",
|
||||
[
|
||||
("DATE_PART('year', d)", exp.Year),
|
||||
("DATE_PART('second', d)", exp.Second),
|
||||
("EXTRACT(year FROM d)", exp.Extract),
|
||||
],
|
||||
)
|
||||
def test_date_part_and_extract(sql, expected_expr):
|
||||
parsed = parse_one(sql, read=Impala)
|
||||
if expected_expr == exp.Extract:
|
||||
assert isinstance(parsed, expected_expr)
|
||||
else:
|
||||
assert isinstance(parsed, expected_expr)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expr, expected_sql",
|
||||
[
|
||||
(
|
||||
exp.DateAdd(
|
||||
this=exp.Column(this="x"),
|
||||
expression=exp.Literal.number(1),
|
||||
unit=exp.Literal.string("YEAR"),
|
||||
),
|
||||
"YEARS_ADD(x, 1)",
|
||||
),
|
||||
(
|
||||
exp.DateSub(
|
||||
this=exp.Column(this="y"),
|
||||
expression=exp.Literal.number(2),
|
||||
unit=exp.Literal.string("MONTH"),
|
||||
),
|
||||
"MONTHS_SUB(y, 2)",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sql_generation_for_date_add_sub(expr, expected_sql):
|
||||
sql = expr.sql(dialect=Impala)
|
||||
assert sql == expected_sql
|
||||
|
||||
|
||||
def test_string_type_is_mapped():
|
||||
expr = exp.DataType.build("VARCHAR")
|
||||
sql = expr.sql(dialect=Impala)
|
||||
assert sql == "STRING"
|
||||
|
||||
|
||||
def test_unsupported_functions_raise():
|
||||
gen = Impala.Generator()
|
||||
with pytest.raises(Exception):
|
||||
gen.sql(exp.StrToMap(this=exp.Literal.string("x")))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sql",
|
||||
[
|
||||
"LATERAL VIEW explode(arr) t",
|
||||
],
|
||||
)
|
||||
def test_lateral_not_supported(sql):
|
||||
parsed = parse_one(sql, read=Impala)
|
||||
assert parsed is None or isinstance(parsed, exp.Expression)
|
||||
Reference in New Issue
Block a user