Compare commits

...

5 Commits

Author SHA1 Message Date
Beto Dealmeida
a5fb5dcd54 feat: Impala dialect for sqlglot 2025-07-28 16:40:46 -04:00
Maxime Beauchemin
604d72cc98 feat: introducing a docker-compose-light.yml for lighter development (#34324) 2025-07-28 09:27:07 -07:00
Enzo Martellucci
913e068113 style(FastVizSwitcher): Adjust padding for FastVizSwitcher selector (#34317) 2025-07-28 14:39:10 +03:00
Geido
1a4e2173f5 fix(NavBar): Add brand text back (#34318) 2025-07-28 12:19:14 +03:00
Ian McEwen
c49789167b style(chart): restyle table pagination (#34311) 2025-07-27 19:39:10 -07:00
21 changed files with 889 additions and 24 deletions

View File

@@ -20,6 +20,9 @@
# If you choose to use this type of deployment make sure to
# create you own docker environment file (docker/.env) with your own
# unique random secure passwords and SECRET_KEY.
#
# For verbose logging during development:
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
# -----------------------------------------------------------------------
x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev}
x-superset-volumes:

157
docker-compose-light.yml Normal file
View File

@@ -0,0 +1,157 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -----------------------------------------------------------------------
# Lightweight docker-compose for running multiple Superset instances
# This includes only essential services: database, Redis, and Superset app
#
# IMPORTANT: To run multiple instances in parallel:
# - Use different project names: docker-compose -p project1 -f docker-compose-light.yml up
# - Use different NODE_PORT values: NODE_PORT=9002 docker-compose -p project2 -f docker-compose-light.yml up
# - Volumes are isolated by project name (e.g., project1_db_home_light, project2_db_home_light)
# - Database name is intentionally different (superset_light) to prevent accidental cross-connections
#
# For verbose logging during development:
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
# -----------------------------------------------------------------------
x-superset-user: &superset-user root
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
- ./superset:/app/superset
- ./superset-frontend:/app/superset-frontend
- superset_home_light:/app/superset_home
- ./tests:/app/tests
x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
cache_from:
- apache/superset-cache:3.10-slim-bookworm
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
INCLUDE_FIREFOX: ${INCLUDE_FIREFOX:-false}
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
services:
db-light:
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
image: postgres:16
restart: unless-stopped
# No host port mapping - only accessible within Docker network
volumes:
- db_home_light:/var/lib/postgresql/data
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
environment:
# Override database name to avoid conflicts
POSTGRES_DB: superset_light
superset-light:
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
build:
<<: *common-build
command: ["/app/docker/docker-bootstrap.sh", "app"]
restart: unless-stopped
# No host port mapping - accessed via webpack dev server proxy
extra_hosts:
- "host.docker.internal:host-gateway"
user: *superset-user
depends_on:
superset-init-light:
condition: service_completed_successfully
volumes: *superset-volumes
environment:
# Override DB connection for light service
DATABASE_HOST: db-light
DATABASE_DB: superset_light
POSTGRES_DB: superset_light
EXAMPLES_HOST: db-light
EXAMPLES_DB: superset_light
EXAMPLES_USER: superset
EXAMPLES_PASSWORD: superset
# Use light-specific config that disables Redis
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
superset-init-light:
build:
<<: *common-build
command: ["/app/docker/docker-init.sh"]
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
depends_on:
db-light:
condition: service_started
user: *superset-user
volumes: *superset-volumes
environment:
# Override DB connection for light service
DATABASE_HOST: db-light
DATABASE_DB: superset_light
POSTGRES_DB: superset_light
EXAMPLES_HOST: db-light
EXAMPLES_DB: superset_light
EXAMPLES_USER: superset
EXAMPLES_PASSWORD: superset
# Use light-specific config that disables Redis
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
healthcheck:
disable: true
superset-node-light:
build:
context: .
target: superset-node
args:
# This prevents building the frontend bundle since we'll mount local folder
# and build it on startup while firing docker-frontend.sh in dev mode, where
# it'll mount and watch local files and rebuild as you update them
DEV_MODE: "true"
BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false}
environment:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
NPM_RUN_PRUNE: false
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
# configuring the dev-server to use the host.docker.internal to connect to the backend
superset: "http://superset-light:8088"
ports:
- "127.0.0.1:${NODE_PORT:-9001}:9000" # Parameterized port
command: ["/app/docker/docker-frontend.sh"]
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
volumes: *superset-volumes
volumes:
superset_home_light:
external: false
db_home_light:
external: false

View File

@@ -20,6 +20,9 @@
# If you choose to use this type of deployment make sure to
# create you own docker environment file (docker/.env) with your own
# unique random secure passwords and SECRET_KEY.
#
# For verbose logging during development:
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
# -----------------------------------------------------------------------
x-superset-volumes:
&superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container

View File

@@ -20,6 +20,9 @@
# If you choose to use this type of deployment make sure to
# create you own docker environment file (docker/.env) with your own
# unique random secure passwords and SECRET_KEY.
#
# For verbose logging during development:
# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs
# -----------------------------------------------------------------------
x-superset-user: &superset-user root
x-superset-volumes: &superset-volumes

View File

@@ -53,7 +53,12 @@ PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
REDIS_HOST=redis
REDIS_PORT=6379
# Development and logging configuration
# FLASK_DEBUG: Enables Flask dev features (auto-reload, better error pages) - keep 'true' for development
FLASK_DEBUG=true
# SUPERSET_LOG_LEVEL: Controls Superset application logging verbosity (debug, info, warning, error, critical)
SUPERSET_LOG_LEVEL=info
SUPERSET_APP_ROOT="/"
SUPERSET_ENV=development
SUPERSET_LOAD_EXAMPLES=yes
@@ -66,4 +71,3 @@ SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
ENABLE_PLAYWRIGHT=false
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
SUPERSET_LOG_LEVEL=info

View File

@@ -20,4 +20,5 @@
# DON'T ignore the .gitignore
!.gitignore
!superset_config.py
!superset_config_docker_light.py
!superset_config_local.example

View File

@@ -129,7 +129,7 @@ if os.getenv("CYPRESS_CONFIG") == "true":
#
try:
import superset_config_docker
from superset_config_docker import * # noqa
from superset_config_docker import * # noqa: F403
logger.info(
f"Loaded your Docker configuration at [{superset_config_docker.__file__}]"

View File

@@ -0,0 +1,37 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Configuration for docker-compose-light.yml - disables Redis and uses minimal services
# Import all settings from the main config first
from flask_caching.backends.filesystemcache import FileSystemCache
from superset_config import * # noqa: F403
# Override caching to use simple in-memory cache instead of Redis
RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
CACHE_CONFIG = {
"CACHE_TYPE": "SimpleCache",
"CACHE_DEFAULT_TIMEOUT": 300,
"CACHE_KEY_PREFIX": "superset_light_",
}
DATA_CACHE_CONFIG = CACHE_CONFIG
THUMBNAIL_CACHE_CONFIG = CACHE_CONFIG
# Disable Celery entirely for lightweight mode
CELERY_CONFIG = None # type: ignore[assignment,misc]

View File

@@ -26,11 +26,14 @@ Superset locally is using Docker Compose on a Linux or Mac OSX
computer. Superset does not have official support for Windows. It's also the easiest
way to launch a fully functioning **development environment** quickly.
Note that there are 3 major ways we support to run `docker compose`:
Note that there are 4 major ways we support to run `docker compose`:
1. **docker-compose.yml:** for interactive development, where we mount your local folder with the
frontend/backend files that you can edit and experience the changes you
make in the app in real time
1. **docker-compose-light.yml:** a lightweight configuration with minimal services (database,
Superset app, and frontend dev server) for development. Uses in-memory caching instead of Redis
and is designed for running multiple instances simultaneously
1. **docker-compose-non-dev.yml** where we just build a more immutable image based on the
local branch and get all the required images running. Changes in the local branch
at the time you fire this up will be reflected, but changes to the code
@@ -44,7 +47,7 @@ Note that there are 3 major ways we support to run `docker compose`:
The `dev` builds include the `psycopg2-binary` required to connect
to the Postgres database launched as part of the `docker compose` builds.
More on these two approaches after setting up the requirements for either.
More on these approaches after setting up the requirements for either.
## Requirements
@@ -103,13 +106,36 @@ and help you start fresh. In the context of `docker compose` setting
from within docker. This will slow down the startup, but will fix various npm-related issues.
:::
### Option #2 - build a set of immutable images from the local branch
### Option #2 - lightweight development with multiple instances
For a lighter development setup that uses fewer resources and supports running multiple instances:
```bash
# Single lightweight instance (default port 9001)
docker compose -f docker-compose-light.yml up
# Multiple instances with different ports
NODE_PORT=9001 docker compose -p superset-1 -f docker-compose-light.yml up
NODE_PORT=9002 docker compose -p superset-2 -f docker-compose-light.yml up
NODE_PORT=9003 docker compose -p superset-3 -f docker-compose-light.yml up
```
This configuration includes:
- PostgreSQL database (internal network only)
- Superset application server
- Frontend development server with webpack hot reloading
- In-memory caching (no Redis)
- Isolated volumes and networks per instance
Access each instance at `http://localhost:{NODE_PORT}` (e.g., `http://localhost:9001`).
### Option #3 - build a set of immutable images from the local branch
```bash
docker compose -f docker-compose-non-dev.yml up
```
### Option #3 - boot up an official release
### Option #4 - boot up an official release
```bash
# Set the version you want to run

View File

@@ -149,7 +149,12 @@ export default styled.div`
.dt-pagination {
text-align: right;
/* use padding instead of margin so clientHeight can capture it */
padding-top: 0.5em;
padding: ${theme.paddingXXS}px 0px;
}
.dt-pagination .pagination > li {
display: inline;
margin: 0 ${theme.marginXXS}px;
}
.dt-pagination .pagination > li > a,
@@ -157,6 +162,8 @@ export default styled.div`
background-color: ${theme.colorBgBase};
color: ${theme.colorText};
border-color: ${theme.colorBorderSecondary};
padding: ${theme.paddingXXS}px ${theme.paddingXS}px;
border-radius: ${theme.borderRadius}px;
}
.dt-pagination .pagination > li.active > a,

View File

@@ -96,7 +96,6 @@ const StyledTabsContainer = styled.div`
.ant-tabs-content-holder {
overflow: visible;
padding-top: ${theme.sizeUnit * 4}px;
}
}

View File

@@ -19,7 +19,7 @@
import { memo, useMemo } from 'react';
import { useSelector } from 'react-redux';
import { css, SupersetTheme } from '@superset-ui/core';
import { Icons } from '@superset-ui/core/components/Icons';
import { Flex, Icons } from '@superset-ui/core/components';
import { getChartKey } from 'src/explore/exploreUtils';
import { ExplorePageState } from 'src/explore/types';
import { FastVizSwitcherProps } from './types';
@@ -79,14 +79,7 @@ export const FastVizSwitcher = memo(
}, [currentSelection, currentViz]);
return (
<div
css={(theme: SupersetTheme) => css`
display: flex;
justify-content: space-between;
column-gap: ${theme.sizeUnit}px;
`}
data-test="fast-viz-switcher"
>
<Flex justify="space-between" gap={4} data-test="fast-viz-switcher">
{vizTiles.map(vizMeta => (
<VizTile
vizMeta={vizMeta}
@@ -96,7 +89,7 @@ export const FastVizSwitcher = memo(
key={vizMeta.name}
/>
))}
</div>
</Flex>
);
},
);

View File

@@ -124,7 +124,7 @@ export const VizTile = ({
>
<span
css={css`
padding: 0px ${theme.sizeUnit}px;
padding: 0px ${theme.sizeUnit * 1.25}px;
`}
>
{vizMeta.icon}
@@ -136,6 +136,7 @@ export const VizTile = ({
font-size: ${theme.fontSizeSM}px;
min-width: 0;
padding-right: ${theme.sizeUnit}px;
line-height: 1;
`}
ref={chartNameRef}
>

View File

@@ -612,3 +612,42 @@ test('should render an extension component if one is supplied', async () => {
expect(extension[0]).toBeInTheDocument();
});
test('should render the brand text if available', async () => {
useSelectorMock.mockReturnValue({ roles: [] });
const modifiedProps = {
...mockedProps,
data: {
...mockedProps.data,
brand: {
...mockedProps.data.brand,
text: 'Welcome to Superset',
},
},
};
render(<Menu {...modifiedProps} />, {
useRouter: true,
useQueryParams: true,
useRedux: true,
useTheme: true,
});
const brandText = await screen.findByText('Welcome to Superset');
expect(brandText).toBeInTheDocument();
});
test('should not render the brand text if not available', async () => {
useSelectorMock.mockReturnValue({ roles: [] });
const text = 'Welcome to Superset';
render(<Menu {...mockedProps} />, {
useRouter: true,
useQueryParams: true,
useRedux: true,
useTheme: true,
});
const brandText = screen.queryByText(text);
expect(brandText).not.toBeInTheDocument();
});

View File

@@ -52,6 +52,8 @@ const StyledHeader = styled.header`
display: none;
}
& .ant-image{
display: contents;
height: 100%;
padding: ${theme.sizeUnit}px
${theme.sizeUnit * 2}px
${theme.sizeUnit}px
@@ -87,7 +89,7 @@ const StyledHeader = styled.header`
padding-left: ${theme.sizeUnit * 4}px;
padding-right: ${theme.sizeUnit * 4}px;
margin-right: ${theme.sizeUnit * 6}px;
font-size: ${theme.sizeUnit * 4}px;
font-size: ${theme.fontSizeLG}px;
float: left;
display: flex;
flex-direction: column;
@@ -322,6 +324,11 @@ export function Menu({
>
{renderBrand()}
</Tooltip>
{brand.text && (
<div className="navbar-brand-text">
<span>{brand.text}</span>
</div>
)}
<MainNav
mode={showMenu}
data-test="navbar-top"

View File

@@ -53,6 +53,7 @@ const {
measure = false,
nameChunks = false,
} = parsedArgs;
const isDevMode = mode !== 'production';
const isDevServer = process.argv[1].includes('webpack-dev-server');
@@ -535,6 +536,11 @@ if (isDevMode) {
runtimeErrors: error => !/ResizeObserver/.test(error.message),
},
logging: 'error',
webSocketURL: {
hostname: '0.0.0.0',
pathname: '/ws',
port: 0,
},
},
static: {
directory: path.join(process.cwd(), '../static/assets'),

View File

@@ -1155,7 +1155,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods
}
CELERY_CONFIG: type[CeleryConfig] = CeleryConfig
CELERY_CONFIG: type[CeleryConfig] | None = CeleryConfig
# Set celery config to None to disable all the above configuration
# CELERY_CONFIG = None

View File

@@ -17,5 +17,6 @@
from .dremio import Dremio
from .firebolt import Firebolt, FireboltOld
from .impala import Impala
__all__ = ["Dremio", "Firebolt", "FireboltOld"]
__all__ = ["Dremio", "Firebolt", "FireboltOld", "Impala"]

View File

@@ -0,0 +1,259 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations
import typing as t
from sqlglot import exp, generator, parser
from sqlglot.dialects.hive import Hive
from sqlglot.helper import seq_get
class Impala(Hive):
"""
A sqlglot dialect for Impala.
Impala is similar to Hive but with some key differences:
- No support for LATERAL VIEW, use JOIN with UNNEST instead
- Different date/time functions
- No support for TRANSFORM
- Limited support for certain Hive-specific features
"""
class Parser(Hive.Parser):
FUNCTIONS = {
**Hive.Parser.FUNCTIONS,
# Impala-specific functions
"MONTHS_ADD": lambda args: exp.DateAdd(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("MONTH"),
),
"MONTHS_SUB": lambda args: exp.DateSub(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("MONTH"),
),
"YEARS_ADD": lambda args: exp.DateAdd(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("YEAR"),
),
"YEARS_SUB": lambda args: exp.DateSub(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("YEAR"),
),
"DAYS_ADD": lambda args: exp.DateAdd(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("DAY"),
),
"DAYS_SUB": lambda args: exp.DateSub(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("DAY"),
),
"WEEKS_ADD": lambda args: exp.DateAdd(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("WEEK"),
),
"WEEKS_SUB": lambda args: exp.DateSub(
this=seq_get(args, 0),
expression=seq_get(args, 1),
unit=exp.Literal.string("WEEK"),
),
# Impala uses different names for some functions
"DATE_PART": lambda args: _parse_date_part(args),
"EXTRACT": lambda args: _parse_extract(args),
# Override Hive functions that Impala doesn't support
"STR_TO_MAP": None, # Not supported in Impala
"XPATH": None, # Not supported in Impala
"XPATH_BOOLEAN": None,
"XPATH_DOUBLE": None,
"XPATH_FLOAT": None,
"XPATH_INT": None,
"XPATH_LONG": None,
"XPATH_SHORT": None,
"XPATH_STRING": None,
}
NO_PAREN_FUNCTION_PARSERS = {
**parser.Parser.NO_PAREN_FUNCTION_PARSERS,
# Remove TRANSFORM as it's not supported in Impala
}
NO_PAREN_FUNCTION_PARSERS.pop("TRANSFORM", None)
def _parse_lateral(self) -> t.Optional[exp.Lateral]:
# Impala doesn't support LATERAL VIEW, it uses different syntax
# This prevents parsing LATERAL VIEW syntax
return None
class Generator(Hive.Generator):
# Impala-specific type mappings
TYPE_MAPPING = {
**Hive.Generator.TYPE_MAPPING,
exp.DataType.Type.VARCHAR: "STRING", # Impala treats VARCHAR as STRING
exp.DataType.Type.NVARCHAR: "STRING",
exp.DataType.Type.CHAR: "STRING", # Impala treats CHAR as STRING
exp.DataType.Type.NCHAR: "STRING",
}
TRANSFORMS = {
**Hive.Generator.TRANSFORMS,
# Date/time functions
exp.DateAdd: lambda self, e: _date_add_sql(self, e),
exp.DateSub: lambda self, e: _date_sub_sql(self, e),
# Impala doesn't support certain Hive features
exp.StrToMap: lambda self, e: self.unsupported(
"STR_TO_MAP is not supported in Impala"
),
exp.Transform: lambda self, e: self.unsupported(
"TRANSFORM is not supported in Impala"
),
exp.QueryTransform: lambda self, e: self.unsupported(
"TRANSFORM is not supported in Impala"
),
# Override LATERAL VIEW handling
exp.Lateral: lambda self, e: _lateral_sql(self, e),
# JSON functions have different names in Impala
exp.JSONExtract: lambda self, e: self.func(
"JSON_QUERY", e.this, e.expression
),
exp.JSONExtractScalar: lambda self, e: self.func(
"JSON_VALUE", e.this, e.expression
),
# Impala uses different syntax for COLLECT_LIST/SET
exp.ArrayAgg: lambda self, e: self.func(
"GROUP_CONCAT",
e.this.this if isinstance(e.this, exp.Order) else e.this,
exp.Literal.string(","),
),
exp.ArrayUniqueAgg: lambda self, e: self.func(
"GROUP_CONCAT",
self.sql(exp.Distinct(expressions=[e.this])),
exp.Literal.string(","),
),
}
def datatype_sql(self, expression: exp.DataType) -> str:
# Impala treats CHAR/VARCHAR as STRING
if expression.is_type("char", "varchar", "nchar", "nvarchar"):
return "STRING"
return super().datatype_sql(expression)
def lateral_sql(self, expression: exp.Lateral) -> str:
# Impala doesn't use LATERAL VIEW syntax
# Instead, it uses regular JOIN with UNNEST
if isinstance(expression.this, exp.Unnest):
return self.sql(expression.this)
return super().lateral_sql(expression)
def _parse_date_part(args: t.List[exp.Expression]) -> exp.Expression:
"""Parse DATE_PART function which extracts date parts."""
part = seq_get(args, 0)
date = seq_get(args, 1)
if isinstance(part, exp.Literal):
part_name = part.name.upper()
if part_name == "YEAR":
return exp.Year(this=date)
elif part_name == "MONTH":
return exp.Month(this=date)
elif part_name == "DAY":
return exp.Day(this=date)
elif part_name == "HOUR":
return exp.Hour(this=date)
elif part_name == "MINUTE":
return exp.Minute(this=date)
elif part_name == "SECOND":
return exp.Second(this=date)
return exp.Extract(this=part, expression=date)
def _parse_extract(args: t.List[exp.Expression]) -> exp.Expression:
"""Parse EXTRACT function."""
return exp.Extract(this=seq_get(args, 0), expression=seq_get(args, 1))
def _date_add_sql(self: generator.Generator, expression: exp.DateAdd) -> str:
"""Generate SQL for date addition in Impala."""
unit = expression.text("unit").upper()
# Map generic units to Impala-specific functions
unit_map = {
"YEAR": "YEARS_ADD",
"MONTH": "MONTHS_ADD",
"WEEK": "WEEKS_ADD",
"DAY": "DAYS_ADD",
}
func_name = unit_map.get(unit, "DATE_ADD")
if func_name != "DATE_ADD":
return self.func(func_name, expression.this, expression.expression)
# For other units, use DATE_ADD with INTERVAL
return self.func(
"DATE_ADD",
expression.this,
self.sql(exp.Interval(this=expression.expression, unit=expression.unit)),
)
def _date_sub_sql(self: generator.Generator, expression: exp.DateSub) -> str:
"""Generate SQL for date subtraction in Impala."""
unit = expression.text("unit").upper()
# Map generic units to Impala-specific functions
unit_map = {
"YEAR": "YEARS_SUB",
"MONTH": "MONTHS_SUB",
"WEEK": "WEEKS_SUB",
"DAY": "DAYS_SUB",
}
func_name = unit_map.get(unit, "DATE_SUB")
if func_name != "DATE_SUB":
return self.func(func_name, expression.this, expression.expression)
# For other units, use DATE_SUB with INTERVAL
return self.func(
"DATE_SUB",
expression.this,
self.sql(exp.Interval(this=expression.expression, unit=expression.unit)),
)
def _lateral_sql(self: generator.Generator, expression: exp.Lateral) -> str:
"""Generate SQL for LATERAL expressions in Impala."""
# Impala doesn't support LATERAL VIEW syntax
# It uses regular JOIN with UNNEST instead
this = expression.this
if isinstance(this, exp.Unnest):
# Just return the UNNEST expression without LATERAL VIEW
return self.sql(this)
# For other cases, try to generate standard SQL
return self.sql(this)

View File

@@ -44,7 +44,7 @@ from sqlglot.optimizer.scope import (
)
from superset.exceptions import QueryClauseValidationException, SupersetParseError
from superset.sql.dialects import Dremio, Firebolt
from superset.sql.dialects import Dremio, Firebolt, Impala
if TYPE_CHECKING:
from superset.models.core import Database
@@ -81,7 +81,7 @@ SQLGLOT_DIALECTS = {
"hana": Dialects.POSTGRES,
"hive": Dialects.HIVE,
# "ibmi": ???
# "impala": ???
"impala": Impala,
# "kustosql": ???
# "kylin": ???
"mariadb": Dialects.MYSQL,

View File

@@ -0,0 +1,319 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from sqlglot import exp, parse_one
from superset.sql.dialects.impala import Impala
def test_impala_date_add_functions() -> None:
"""
Test Impala-specific date addition functions.
"""
# Test MONTHS_ADD
sql = "SELECT MONTHS_ADD(date_col, 3) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT MONTHS_ADD(date_col, 3) FROM table1"
# Test YEARS_ADD
sql = "SELECT YEARS_ADD(date_col, 2) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT YEARS_ADD(date_col, 2) FROM table1"
# Test DAYS_ADD
sql = "SELECT DAYS_ADD(date_col, 7) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT DAYS_ADD(date_col, 7) FROM table1"
# Test WEEKS_ADD
sql = "SELECT WEEKS_ADD(date_col, 4) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT WEEKS_ADD(date_col, 4) FROM table1"
def test_impala_date_sub_functions() -> None:
"""
Test Impala-specific date subtraction functions.
"""
# Test MONTHS_SUB
sql = "SELECT MONTHS_SUB(date_col, 3) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT MONTHS_SUB(date_col, 3) FROM table1"
# Test YEARS_SUB
sql = "SELECT YEARS_SUB(date_col, 2) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT YEARS_SUB(date_col, 2) FROM table1"
# Test DAYS_SUB
sql = "SELECT DAYS_SUB(date_col, 7) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT DAYS_SUB(date_col, 7) FROM table1"
# Test WEEKS_SUB
sql = "SELECT WEEKS_SUB(date_col, 4) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT WEEKS_SUB(date_col, 4) FROM table1"
def test_impala_date_part_function() -> None:
"""
Test DATE_PART function parsing.
"""
# Test DATE_PART with YEAR
sql = "SELECT DATE_PART('YEAR', date_col) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT YEAR(date_col) FROM table1"
# Test DATE_PART with MONTH
sql = "SELECT DATE_PART('MONTH', date_col) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT MONTH(date_col) FROM table1"
# Test DATE_PART with DAY
sql = "SELECT DATE_PART('DAY', date_col) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT DAY(date_col) FROM table1"
def test_impala_data_types() -> None:
"""
Test that Impala treats VARCHAR/CHAR as STRING.
"""
# Test VARCHAR conversion
sql = "CREATE TABLE test (col1 VARCHAR(100))"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "CREATE TABLE test (col1 STRING)"
# Test CHAR conversion
sql = "CREATE TABLE test (col1 CHAR(10))"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "CREATE TABLE test (col1 STRING)"
# Test NVARCHAR conversion
sql = "CREATE TABLE test (col1 NVARCHAR(50))"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "CREATE TABLE test (col1 STRING)"
def test_impala_unsupported_functions() -> None:
"""
Test that unsupported Hive functions are handled properly.
"""
# STR_TO_MAP is not supported
with pytest.raises(Exception) as exc_info:
sql = "SELECT STR_TO_MAP('a:1,b:2', ',', ':') FROM table1"
ast = parse_one(sql, dialect=Impala)
ast.sql(dialect=Impala)
assert "STR_TO_MAP is not supported" in str(exc_info.value)
# TRANSFORM is not supported
with pytest.raises(Exception) as exc_info:
sql = "SELECT TRANSFORM(col) USING 'script.py' FROM table1"
ast = parse_one(sql, dialect=Impala)
ast.sql(dialect=Impala)
assert "TRANSFORM is not supported" in str(exc_info.value)
def test_impala_json_functions() -> None:
"""
Test that JSON functions are mapped to Impala equivalents.
"""
# Test JSON extract scalar to JSON_VALUE
sql = "SELECT GET_JSON_OBJECT(json_col, '$.field') FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT JSON_VALUE(json_col, '$.field') FROM table1"
def test_impala_aggregate_functions() -> None:
"""
Test that COLLECT_LIST/SET are mapped to GROUP_CONCAT.
"""
# Test COLLECT_LIST conversion
sql = "SELECT COLLECT_LIST(col1) FROM table1 GROUP BY col2"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT GROUP_CONCAT(col1, ',') FROM table1 GROUP BY col2"
# Test COLLECT_SET conversion
sql = "SELECT COLLECT_SET(col1) FROM table1 GROUP BY col2"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert (
regenerated
== "SELECT GROUP_CONCAT(DISTINCT col1, ',') FROM table1 GROUP BY col2"
)
def test_standard_date_add_sub() -> None:
"""
Test standard DATE_ADD/DATE_SUB with generic units.
"""
# Generic DATE_ADD (should be parsed and regenerated as-is)
sql = "SELECT DATE_ADD(date_col, 5) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT DAYS_ADD(date_col, 5) FROM table1"
def test_unnest_without_lateral_view() -> None:
"""
Test that UNNEST is handled without LATERAL VIEW syntax.
"""
# Impala uses JOIN with UNNEST instead of LATERAL VIEW
sql = "SELECT * FROM table1, UNNEST(array_col) AS t(elem)"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
# Should not contain LATERAL VIEW
assert "LATERAL VIEW" not in regenerated
assert "UNNEST" in regenerated
def test_extract_function() -> None:
"""
Test EXTRACT function.
"""
sql = "SELECT EXTRACT(YEAR FROM date_col) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT EXTRACT(YEAR FROM date_col) FROM table1"
sql = "SELECT EXTRACT(MONTH FROM date_col) FROM table1"
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
assert regenerated == "SELECT EXTRACT(MONTH FROM date_col) FROM table1"
def test_complex_query() -> None:
"""
Test a more complex query with multiple Impala-specific features.
"""
sql = """
SELECT
customer_id,
GROUP_CONCAT(product_name, ',') as products,
MONTHS_ADD(order_date, 1) as next_month,
JSON_VALUE(order_details, '$.total') as total
FROM orders
WHERE YEARS_SUB(order_date, 1) > '2022-01-01'
GROUP BY customer_id, order_date, order_details
"""
ast = parse_one(sql, dialect=Impala)
regenerated = ast.sql(dialect=Impala)
# Check key components are preserved
assert "GROUP_CONCAT" in regenerated
assert "MONTHS_ADD" in regenerated
assert "JSON_VALUE" in regenerated
assert "YEARS_SUB" in regenerated
@pytest.mark.parametrize(
"func, expected_class, unit",
[
("MONTHS_ADD(x, 1)", exp.DateAdd, "MONTH"),
("YEARS_SUB(x, 2)", exp.DateSub, "YEAR"),
("DAYS_ADD(x, 3)", exp.DateAdd, "DAY"),
("WEEKS_SUB(x, 4)", exp.DateSub, "WEEK"),
],
)
def test_date_functions_parse_correctly(func, expected_class, unit):
parsed = parse_one(func, read=Impala)
assert isinstance(parsed, expected_class)
assert parsed.text("unit").upper() == unit
@pytest.mark.parametrize(
"sql, expected_expr",
[
("DATE_PART('year', d)", exp.Year),
("DATE_PART('second', d)", exp.Second),
("EXTRACT(year FROM d)", exp.Extract),
],
)
def test_date_part_and_extract(sql, expected_expr):
parsed = parse_one(sql, read=Impala)
if expected_expr == exp.Extract:
assert isinstance(parsed, expected_expr)
else:
assert isinstance(parsed, expected_expr)
@pytest.mark.parametrize(
"expr, expected_sql",
[
(
exp.DateAdd(
this=exp.Column(this="x"),
expression=exp.Literal.number(1),
unit=exp.Literal.string("YEAR"),
),
"YEARS_ADD(x, 1)",
),
(
exp.DateSub(
this=exp.Column(this="y"),
expression=exp.Literal.number(2),
unit=exp.Literal.string("MONTH"),
),
"MONTHS_SUB(y, 2)",
),
],
)
def test_sql_generation_for_date_add_sub(expr, expected_sql):
sql = expr.sql(dialect=Impala)
assert sql == expected_sql
def test_string_type_is_mapped():
expr = exp.DataType.build("VARCHAR")
sql = expr.sql(dialect=Impala)
assert sql == "STRING"
def test_unsupported_functions_raise():
gen = Impala.Generator()
with pytest.raises(Exception):
gen.sql(exp.StrToMap(this=exp.Literal.string("x")))
@pytest.mark.parametrize(
"sql",
[
"LATERAL VIEW explode(arr) t",
],
)
def test_lateral_not_supported(sql):
parsed = parse_one(sql, read=Impala)
assert parsed is None or isinstance(parsed, exp.Expression)