Compare commits

...

4 Commits

Author SHA1 Message Date
Beto Dealmeida
10d148f569 Fix tests 2025-12-19 15:47:06 -05:00
Beto Dealmeida
bf342d66db Fix case 2025-12-19 13:22:58 -05:00
Beto Dealmeida
3e38931a8c feat: file handler for CSV/XSL 2025-12-19 10:58:36 -05:00
Beto Dealmeida
c07382d606 Snowflake fixes 2025-12-19 10:56:43 -05:00
18 changed files with 2383 additions and 42 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 247 KiB

View File

@@ -67,6 +67,7 @@ interface UploadDataModalProps {
show: boolean;
allowedExtensions: string[];
type: UploadType;
fileListOverride?: File[];
}
const CSVSpecificFields = [
@@ -215,6 +216,7 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
show,
allowedExtensions,
type = 'csv',
fileListOverride,
}) => {
const [form] = Form.useForm();
const [currentDatabaseId, setCurrentDatabaseId] = useState<number>(0);
@@ -524,10 +526,26 @@ const UploadDataModal: FunctionComponent<UploadDataModalProps> = ({
await loadFileMetadata(info.file.originFileObj);
};
useEffect(() => {
if (fileListOverride?.length) {
setFileList(
fileListOverride.map(file => ({
uid: file.name,
name: file.name,
originFileObj: file as UploadFile['originFileObj'],
status: 'done' as const,
})),
);
if (previewUploadedFile) {
loadFileMetadata(fileListOverride[0]).then(r => r);
}
}
}, [fileListOverride, previewUploadedFile]);
useEffect(() => {
if (
columns.length > 0 &&
fileList[0].originFileObj &&
fileList.length > 0 &&
fileList[0].originFileObj instanceof File
) {
if (!previewUploadedFile) {

View File

@@ -0,0 +1,368 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { ComponentType } from 'react';
import { render, screen, waitFor } from 'spec/helpers/testing-library';
import { MemoryRouter, Route } from 'react-router-dom';
import FileHandler from './index';
const mockAddDangerToast = jest.fn();
const mockAddSuccessToast = jest.fn();
const mockHistoryPush = jest.fn();
type ToastInjectedProps = {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
};
// Mock the withToasts HOC
jest.mock('src/components/MessageToasts/withToasts', () => ({
__esModule: true,
default: (Component: ComponentType<ToastInjectedProps>) =>
function MockedWithToasts(props: Record<string, unknown>) {
return (
<Component
{...props}
addDangerToast={mockAddDangerToast}
addSuccessToast={mockAddSuccessToast}
/>
);
},
}));
interface UploadDataModalProps {
show: boolean;
onHide: () => void;
type: string;
allowedExtensions: string[];
fileListOverride?: File[];
}
// Mock the UploadDataModal
jest.mock('src/features/databases/UploadDataModel', () => ({
__esModule: true,
default: ({
show,
onHide,
type,
allowedExtensions,
fileListOverride,
}: UploadDataModalProps) => (
<div data-test="upload-modal">
<div data-test="modal-show">{show.toString()}</div>
<div data-test="modal-type">{type}</div>
<div data-test="modal-extensions">{allowedExtensions.join(',')}</div>
<div data-test="modal-file">{fileListOverride?.[0]?.name ?? ''}</div>
<button onClick={onHide}>Close</button>
</div>
),
}));
// Mock react-router-dom's useHistory
jest.mock('react-router-dom', () => ({
...jest.requireActual('react-router-dom'),
useHistory: () => ({
push: mockHistoryPush,
}),
}));
// Mock the File API
type MockFileHandle = {
kind: 'file';
name: string;
getFile: () => Promise<File>;
isSameEntry: () => Promise<boolean>;
queryPermission: () => Promise<PermissionState>;
requestPermission: () => Promise<PermissionState>;
};
const createMockFileHandle = (fileName: string): MockFileHandle => ({
kind: 'file',
name: fileName,
getFile: async () => new File(['test'], fileName),
isSameEntry: async () => false,
queryPermission: async () => 'granted',
requestPermission: async () => 'granted',
});
type LaunchQueue = {
setConsumer: (
consumer: (params: { files?: MockFileHandle[] }) => void,
) => void;
};
const setupLaunchQueue = (fileHandle: MockFileHandle | null = null) => {
let savedConsumer:
| ((params: { files?: MockFileHandle[] }) => void | Promise<void>)
| null = null;
(window as unknown as Window & { launchQueue: LaunchQueue }).launchQueue = {
setConsumer: (consumer: (params: { files?: MockFileHandle[] }) => void) => {
savedConsumer = consumer;
if (fileHandle) {
setTimeout(() => {
consumer({
files: [fileHandle],
});
}, 0);
}
},
};
return {
triggerConsumer: async (params: { files?: MockFileHandle[] }) => {
await savedConsumer?.(params);
},
};
};
beforeEach(() => {
jest.clearAllMocks();
delete (window as any).launchQueue;
});
test('shows error when launchQueue is not supported', async () => {
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
await waitFor(() => {
expect(mockAddDangerToast).toHaveBeenCalledWith(
'File handling is not supported in this browser. Please use a modern browser like Chrome or Edge.',
);
expect(mockHistoryPush).toHaveBeenCalledWith('/superset/welcome/');
});
});
test('redirects when no files are provided', async () => {
const { triggerConsumer } = setupLaunchQueue();
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
// Trigger the consumer with no files
await triggerConsumer({ files: [] });
await waitFor(() => {
expect(mockHistoryPush).toHaveBeenCalledWith('/superset/welcome/');
});
});
test('handles CSV file correctly', async () => {
const fileHandle = createMockFileHandle('test.csv');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
expect(screen.getByTestId('modal-show')).toHaveTextContent('true');
expect(screen.getByTestId('modal-type')).toHaveTextContent('csv');
expect(screen.getByTestId('modal-extensions')).toHaveTextContent('csv');
expect(screen.getByTestId('modal-file')).toHaveTextContent('test.csv');
});
test('handles Excel (.xls) file correctly', async () => {
const fileHandle = createMockFileHandle('test.xls');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
expect(screen.getByTestId('modal-type')).toHaveTextContent('excel');
expect(screen.getByTestId('modal-extensions')).toHaveTextContent('xls,xlsx');
});
test('handles Excel (.xlsx) file correctly', async () => {
const fileHandle = createMockFileHandle('test.xlsx');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
expect(screen.getByTestId('modal-type')).toHaveTextContent('excel');
expect(screen.getByTestId('modal-extensions')).toHaveTextContent('xls,xlsx');
});
test('handles Parquet file correctly', async () => {
const fileHandle = createMockFileHandle('test.parquet');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
expect(screen.getByTestId('modal-type')).toHaveTextContent('columnar');
expect(screen.getByTestId('modal-extensions')).toHaveTextContent('parquet');
});
test('shows error for unsupported file type', async () => {
const { triggerConsumer } = setupLaunchQueue();
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
// Trigger with unsupported file
const fileHandle = createMockFileHandle('test.pdf');
await triggerConsumer({ files: [fileHandle] });
await waitFor(() => {
expect(mockAddDangerToast).toHaveBeenCalledWith(
'Unsupported file type. Please use CSV, Excel, or Columnar files.',
);
expect(mockHistoryPush).toHaveBeenCalledWith('/superset/welcome/');
});
});
test('handles file with uppercase extension', async () => {
const fileHandle = createMockFileHandle('test.CSV');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
expect(screen.getByTestId('modal-type')).toHaveTextContent('csv');
});
test('handles errors during file processing', async () => {
const { triggerConsumer } = setupLaunchQueue();
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
// Trigger with a file handle that throws an error
const errorFileHandle: MockFileHandle = {
kind: 'file',
name: 'error.csv',
getFile: async () => {
throw new Error('File access denied');
},
isSameEntry: async () => false,
queryPermission: async () => 'granted',
requestPermission: async () => 'granted',
};
await triggerConsumer({ files: [errorFileHandle] });
await waitFor(() => {
expect(mockAddDangerToast).toHaveBeenCalledWith(
'Failed to open file. Please try again.',
);
expect(mockHistoryPush).toHaveBeenCalledWith('/superset/welcome/');
});
});
test('modal close redirects to welcome page', async () => {
const fileHandle = createMockFileHandle('test.csv');
setupLaunchQueue(fileHandle);
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
const modal = await screen.findByTestId('upload-modal');
expect(modal).toBeInTheDocument();
// Click the close button in the mocked modal
const closeButton = screen.getByRole('button', { name: 'Close' });
closeButton.click();
await waitFor(() => {
expect(mockHistoryPush).toHaveBeenCalledWith('/superset/welcome/');
});
});
test('shows loading state while waiting for file', () => {
setupLaunchQueue();
render(
<MemoryRouter initialEntries={['/superset/file-handler']}>
<Route path="/superset/file-handler">
<FileHandler />
</Route>
</MemoryRouter>,
{ useRedux: true },
);
// Should show loading initially before file is processed
expect(screen.getByRole('status')).toBeInTheDocument();
});

View File

@@ -0,0 +1,138 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { useEffect, useState } from 'react';
import { useHistory } from 'react-router-dom';
import { t } from '@superset-ui/core';
import { Loading } from '@superset-ui/core/components';
import UploadDataModal from 'src/features/databases/UploadDataModel';
import withToasts from 'src/components/MessageToasts/withToasts';
interface FileLaunchParams {
readonly files?: readonly FileSystemFileHandle[];
}
interface LaunchQueue {
setConsumer: (consumer: (params: FileLaunchParams) => void) => void;
}
interface WindowWithLaunchQueue extends Window {
launchQueue?: LaunchQueue;
}
interface FileHandlerProps {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
}
const FileHandler = ({ addDangerToast, addSuccessToast }: FileHandlerProps) => {
const history = useHistory();
const [uploadFile, setUploadFile] = useState<File | null>(null);
const [uploadType, setUploadType] = useState<
'csv' | 'excel' | 'columnar' | null
>(null);
const [showModal, setShowModal] = useState(false);
const [allowedExtensions, setAllowedExtensions] = useState<string[]>([]);
useEffect(() => {
const handleFileLaunch = async () => {
const { launchQueue } = window as WindowWithLaunchQueue;
if (!launchQueue) {
addDangerToast(
t(
'File handling is not supported in this browser. Please use a modern browser like Chrome or Edge.',
),
);
history.push('/superset/welcome/');
return;
}
launchQueue.setConsumer(async (launchParams: FileLaunchParams) => {
if (!launchParams.files || launchParams.files.length === 0) {
history.push('/superset/welcome/');
return;
}
try {
const fileHandle = launchParams.files[0];
const file = await fileHandle.getFile();
const fileName = file.name.toLowerCase();
let type: 'csv' | 'excel' | 'columnar' | null = null;
let extensions: string[] = [];
if (fileName.endsWith('.csv')) {
type = 'csv';
extensions = ['csv'];
} else if (fileName.endsWith('.xls') || fileName.endsWith('.xlsx')) {
type = 'excel';
extensions = ['xls', 'xlsx'];
} else if (fileName.endsWith('.parquet')) {
type = 'columnar';
extensions = ['parquet'];
} else {
addDangerToast(
t(
'Unsupported file type. Please use CSV, Excel, or Columnar files.',
),
);
history.push('/superset/welcome/');
return;
}
setUploadFile(file);
setUploadType(type);
setAllowedExtensions(extensions);
setShowModal(true);
} catch (error) {
console.error('Error handling file launch:', error);
addDangerToast(t('Failed to open file. Please try again.'));
history.push('/superset/welcome/');
}
});
};
handleFileLaunch();
}, [history, addDangerToast]);
const handleModalClose = () => {
setShowModal(false);
setUploadFile(null);
setUploadType(null);
history.push('/superset/welcome/');
};
if (!uploadFile || !uploadType) {
return <Loading />;
}
return (
<UploadDataModal
show={showModal}
onHide={handleModalClose}
fileListOverride={[uploadFile]}
allowedExtensions={allowedExtensions}
type={uploadType}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
/>
);
};
export default withToasts(FileHandler);

View File

@@ -0,0 +1,65 @@
{
"name": "Apache Superset",
"short_name": "Superset",
"description": "Modern data exploration and visualization platform",
"start_url": "/superset/welcome/",
"scope": "/",
"display": "standalone",
"background_color": "#ffffff",
"theme_color": "#20a7c9",
"icons": [
{
"src": "/static/assets/images/pwa/icon-192.png",
"sizes": "192x192",
"type": "image/png",
"purpose": "any"
},
{
"src": "/static/assets/images/pwa/icon-512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "any"
},
{
"src": "/static/assets/images/pwa/icon-192.png",
"sizes": "192x192",
"type": "image/png",
"purpose": "maskable"
},
{
"src": "/static/assets/images/pwa/icon-512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "maskable"
}
],
"screenshots": [
{
"src": "/static/assets/images/pwa/screenshot-wide.png",
"sizes": "1280x720",
"type": "image/png",
"form_factor": "wide",
"label": "Apache Superset Dashboard"
},
{
"src": "/static/assets/images/pwa/screenshot-narrow.png",
"sizes": "540x720",
"type": "image/png",
"form_factor": "narrow",
"label": "Apache Superset Mobile View"
}
],
"file_handlers": [
{
"action": "/superset/file-handler",
"accept": {
"text/csv": [".csv"],
"application/vnd.ms-excel": [".xls"],
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
".xlsx"
],
"application/vnd.apache.parquet": [".parquet"]
}
}
]
}

View File

@@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// Service Worker types (declared locally to avoid polluting global scope)
declare const self: {
skipWaiting(): Promise<void>;
clients: { claim(): Promise<void> };
addEventListener(
type: 'install' | 'activate',
listener: (event: { waitUntil(promise: Promise<unknown>): void }) => void,
): void;
};
self.addEventListener('install', event => {
event.waitUntil(self.skipWaiting());
});
self.addEventListener('activate', event => {
event.waitUntil(self.clients.claim());
});
export {};

View File

@@ -178,6 +178,10 @@ const UserRegistrations = lazy(
),
);
const FileHandler = lazy(
() => import(/* webpackChunkName: "FileHandler" */ 'src/pages/FileHandler'),
);
type Routes = {
path: string;
Component: ComponentType;
@@ -206,6 +210,10 @@ export const routes: Routes = [
path: '/superset/welcome/',
Component: Home,
},
{
path: '/superset/file-handler',
Component: FileHandler,
},
{
path: '/dashboard/list/',
Component: DashboardList,

View File

@@ -71,20 +71,30 @@ const isDevServer = process.argv[1].includes('webpack-dev-server');
// TypeScript checker memory limit (in MB)
const TYPESCRIPT_MEMORY_LIMIT = 4096;
const defaultEntryFilename = isDevMode
? '[name].[contenthash:8].entry.js'
: nameChunks
? '[name].[chunkhash].entry.js'
: '[name].[chunkhash].entry.js';
const defaultChunkFilename = isDevMode
? '[name].[contenthash:8].chunk.js'
: nameChunks
? '[name].[chunkhash].chunk.js'
: '[chunkhash].chunk.js';
const output = {
path: BUILD_DIR,
publicPath: '/static/assets/',
filename: pathData =>
pathData.chunk?.name === 'service-worker'
? '../service-worker.js'
: defaultEntryFilename,
chunkFilename: pathData =>
pathData.chunk?.name === 'service-worker'
? '../service-worker.js'
: defaultChunkFilename,
};
if (isDevMode) {
output.filename = '[name].[contenthash:8].entry.js';
output.chunkFilename = '[name].[contenthash:8].chunk.js';
} else if (nameChunks) {
output.filename = '[name].[chunkhash].entry.js';
output.chunkFilename = '[name].[chunkhash].chunk.js';
} else {
output.filename = '[name].[chunkhash].entry.js';
output.chunkFilename = '[chunkhash].chunk.js';
}
if (!isDevMode) {
output.clean = true;
@@ -139,7 +149,11 @@ const plugins = [
}),
new CopyPlugin({
patterns: ['package.json', { from: 'src/assets/images', to: 'images' }],
patterns: [
'package.json',
{ from: 'src/assets/images', to: 'images' },
{ from: 'src/pwa-manifest.json', to: 'pwa-manifest.json' },
],
}),
// static pages
@@ -184,7 +198,13 @@ if (!process.env.CI) {
// Add React Refresh plugin for development mode
if (isDevMode) {
plugins.push(new ReactRefreshWebpackPlugin());
plugins.push(
new ReactRefreshWebpackPlugin({
// Exclude service worker from React Refresh - it runs in a worker context
// without DOM/window and doesn't need HMR
exclude: /service-worker/,
}),
);
}
if (!isDevMode) {
@@ -300,6 +320,7 @@ const config = {
menu: addPreamble('src/views/menu.tsx'),
spa: addPreamble('/src/views/index.tsx'),
embedded: addPreamble('/src/embedded/index.tsx'),
'service-worker': path.join(APP_DIR, 'src/service-worker.ts'),
},
cache: {
type: 'filesystem', // Enable filesystem caching

View File

@@ -551,7 +551,7 @@ def apply_data_access_rules(
if table_cls:
cls_rules[qualified_table] = table_cls
# Apply CLS first (before RLS) so that hidden columns are removed
# Apply CLS first (before RLS) so that column transformations happen
# before RLS wraps the query in a subquery
if cls_rules:
# Build schema dict for sqlglot's qualify() to expand SELECT *
@@ -591,8 +591,10 @@ def apply_data_access_rules(
parsed_statement.apply_cls(cls_rules, schema=table_schemas if table_schemas else None)
# Apply RLS after CLS - RLS wraps the query in a subquery with SELECT *
# which will pick up the already-transformed columns from CLS
# Apply RLS after CLS
# Note: CLS runs qualify() which may change identifier case for some databases
# (e.g., Snowflake uppercases identifiers). The RLSTransformer normalizes
# table names to lowercase for case-insensitive matching.
if rls_predicates:
parsed_statement.apply_rls(catalog, schema, rls_predicates, method)

View File

@@ -648,7 +648,7 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
def register_request_handlers(self) -> None:
"""Register app-level request handlers"""
from flask import Response
from flask import request, Response
@self.superset_app.after_request
def apply_http_headers(response: Response) -> Response:
@@ -664,6 +664,14 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
for k, v in self.superset_app.config["DEFAULT_HTTP_HEADERS"].items():
if k not in response.headers:
response.headers[k] = v
# Allow service worker to control the root scope for PWA file handling
if (
request.path.endswith("service-worker.js")
and "Service-Worker-Allowed" not in response.headers
):
response.headers["Service-Worker-Allowed"] = "/"
return response
@self.superset_app.after_request

View File

@@ -35,6 +35,7 @@ from sqlglot.dialects.dialect import (
)
from sqlglot.dialects.singlestore import SingleStore
from sqlglot.errors import ParseError
from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
from sqlglot.optimizer.pushdown_predicates import (
pushdown_predicates,
)
@@ -292,19 +293,52 @@ class RLSTransformer:
catalog: str | None,
schema: str | None,
rules: dict[Table, list[exp.Expression]],
dialect: Dialects | type[Dialect] | None = None,
) -> None:
self.catalog = catalog
self.schema = schema
self.rules = rules
self.dialect = dialect
# Normalize table keys using dialect-aware normalization
# This ensures matching works correctly regardless of how the dialect
# handles identifier case (e.g., Snowflake uppercases, Postgres lowercases)
self.rules = {
self._normalize_table(table): predicates
for table, predicates in rules.items()
}
def _normalize_table(self, table: Table) -> Table:
"""
Normalize table identifiers using dialect-aware normalization.
This uses sqlglot's normalize_identifiers to match how the dialect
handles identifier case:
- Snowflake: uppercases unquoted identifiers
- PostgreSQL: lowercases unquoted identifiers
- Quoted identifiers preserve their case
"""
# Create a temporary exp.Table node for normalization
table_exp = exp.Table(
this=exp.Identifier(this=table.table) if table.table else None,
db=exp.Identifier(this=table.schema) if table.schema else None,
catalog=exp.Identifier(this=table.catalog) if table.catalog else None,
)
normalized = normalize_identifiers(table_exp, dialect=self.dialect)
return Table(
table=normalized.name if normalized.name else table.table,
schema=normalized.db if normalized.db else table.schema,
catalog=normalized.catalog if normalized.catalog else table.catalog,
)
def get_predicate(self, table_node: exp.Table) -> exp.Expression | None:
"""
Get the combined RLS predicate for a table.
"""
table = Table(
table_node.name,
table_node.db if table_node.db else self.schema,
table_node.catalog if table_node.catalog else self.catalog,
table = self._normalize_table(
Table(
table_node.name,
table_node.db if table_node.db else self.schema,
table_node.catalog if table_node.catalog else self.catalog,
)
)
if predicates := self.rules.get(table):
return sqlglot.and_(*predicates)
@@ -396,7 +430,19 @@ class RLSAsSubqueryTransformer(RLSTransformer):
if predicate := self.get_predicate(node):
if node.alias:
alias = node.alias
# After qualify(), alias might be a string instead of TableAlias
if isinstance(node.alias, str):
# Check if the original alias was quoted by looking at the SQL
# If the alias contains special chars or is a reserved word, quote it
needs_quoting = (
not node.alias.isidentifier()
or "." in node.alias
)
alias = exp.TableAlias(
this=exp.Identifier(this=node.alias, quoted=needs_quoting)
)
else:
alias = node.alias
else:
name = ".".join(
part
@@ -444,20 +490,47 @@ class CLSTransformer:
rules: CLSRules,
dialect: Dialects | type[Dialect] | None,
) -> None:
self.rules = self._normalize_rules(rules)
self.dialect = dialect
self.rules = self._normalize_rules(rules)
self.hash_pattern = CLS_HASH_FUNCTIONS.get(dialect, CLS_HASH_FUNCTIONS[None])
def _normalize_identifier(self, name: str) -> str:
"""Normalize an identifier using dialect-aware normalization."""
ident = exp.Identifier(this=name)
normalized = normalize_identifiers(ident, dialect=self.dialect)
return normalized.name
def _normalize_table(self, table: Table) -> Table:
"""
Normalize table identifiers using dialect-aware normalization.
This uses sqlglot's normalize_identifiers to match how the dialect
handles identifier case:
- Snowflake: uppercases unquoted identifiers
- PostgreSQL: lowercases unquoted identifiers
- Quoted identifiers preserve their case
"""
table_exp = exp.Table(
this=exp.Identifier(this=table.table) if table.table else None,
db=exp.Identifier(this=table.schema) if table.schema else None,
catalog=exp.Identifier(this=table.catalog) if table.catalog else None,
)
normalized = normalize_identifiers(table_exp, dialect=self.dialect)
return Table(
table=normalized.name if normalized.name else table.table,
schema=normalized.db if normalized.db else table.schema,
catalog=normalized.catalog if normalized.catalog else table.catalog,
)
def _normalize_rules(self, rules: CLSRules) -> dict[Table, dict[str, CLSAction]]:
"""
Normalize table and column names to lowercase for case-insensitive matching.
Normalize table and column names using dialect-aware normalization.
"""
return {
Table(
table=table.table.lower(),
schema=table.schema.lower() if table.schema else None,
catalog=table.catalog.lower() if table.catalog else None,
): {col.lower(): action for col, action in cols.items()}
self._normalize_table(table): {
self._normalize_identifier(col): action
for col, action in cols.items()
}
for table, cols in rules.items()
}
@@ -480,22 +553,24 @@ class CLSTransformer:
return None
# Create a normalized Table for lookup
lookup_table = Table(
table=table_name.lower(),
schema=schema.lower() if schema else None,
catalog=catalog.lower() if catalog else None,
lookup_table = self._normalize_table(
Table(
table=table_name,
schema=schema,
catalog=catalog,
)
)
normalized_column = self._normalize_identifier(column_name)
# First try exact match with schema/catalog
table_rules = self.rules.get(lookup_table)
if table_rules:
return table_rules.get(column_name.lower())
if (table_rules := self.rules.get(lookup_table)):
return table_rules.get(normalized_column)
# Fallback: match by table name only
# This handles cases where the rule has schema/catalog but the query doesn't
for rule_table, cols in self.rules.items():
if rule_table.table == lookup_table.table:
action = cols.get(column_name.lower())
action = cols.get(normalized_column)
if action:
return action
@@ -1506,7 +1581,7 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
if method not in transformers:
raise ValueError(f"Invalid RLS method: {method}")
transformer = transformers[method](catalog, schema, predicates)
transformer = transformers[method](catalog, schema, predicates, self._dialect)
self._parsed = self._parsed.transform(transformer)
def apply_cls(

File diff suppressed because it is too large Load Diff

View File

@@ -28,7 +28,9 @@
{% endblock %}
</title>
{% block head_meta %}{% endblock %}
{% block head_meta %}
<link rel="manifest" href="{{ assets_prefix }}/static/assets/pwa-manifest.json?v=4">
{% endblock %}
<style>
body {
@@ -73,6 +75,17 @@
/>
{% block head_js %}
{% include "head_custom_extra.html" %}
<script nonce="{{ macros.get_nonce() }}">
if ('serviceWorker' in navigator) {
window.addEventListener('load', function() {
navigator.serviceWorker
.register('{{ assets_prefix }}/static/service-worker.js')
.catch(function(err) {
console.error('Service Worker registration failed:', err);
});
});
}
</script>
{% endblock %}
</head>

View File

@@ -908,6 +908,21 @@ class Superset(BaseSupersetView):
return self.render_app_template(extra_bootstrap_data=payload)
@has_access
@event_logger.log_this
@expose("/file-handler")
def file_handler(self) -> FlaskResponse:
"""File handler page for PWA file handling"""
if not g.user or not get_user_id():
return redirect_to_login()
payload = {
"user": bootstrap_user_data(g.user, include_perms=True),
"common": common_bootstrap_payload(),
}
return self.render_app_template(extra_bootstrap_data=payload)
@has_access
@event_logger.log_this
@expose("/sqllab/history/", methods=("GET",))

View File

@@ -3676,10 +3676,13 @@ def test_apply_cls_parametrized(
def test_apply_cls_subquery() -> None:
"""
Test CLS applies to subqueries.
Requires schema to expand SELECT * in outer query.
"""
rules = {Table("users"): {"ssn": CLSAction.HASH}}
schema = {"users": {"ssn": "VARCHAR", "name": "VARCHAR"}}
sql = "SELECT * FROM (SELECT ssn, name FROM users) AS subq"
result = apply_cls(sql, rules, engine="postgresql")
result = apply_cls(sql, rules, engine="postgresql", schema=schema)
assert result == (
"SELECT\n"
@@ -3697,10 +3700,13 @@ def test_apply_cls_subquery() -> None:
def test_apply_cls_cte() -> None:
"""
Test CLS applies to CTEs.
Requires schema to expand SELECT * in outer query.
"""
rules = {Table("users"): {"ssn": CLSAction.HASH}}
schema = {"users": {"ssn": "VARCHAR", "name": "VARCHAR"}}
sql = "WITH cte AS (SELECT ssn, name FROM users) SELECT * FROM cte"
result = apply_cls(sql, rules, engine="postgresql")
result = apply_cls(sql, rules, engine="postgresql", schema=schema)
assert result == (
'WITH "cte" AS (\n'
@@ -4690,3 +4696,106 @@ def test_merge_cls_rules_complex_scenario() -> None:
"credit_card": CLSAction.HIDE,
},
}
def test_combined_rls_and_cls_snowflake() -> None:
"""
Test combined RLS and CLS application with Snowflake dialect.
This tests a real-world scenario where:
- A table has both RLS (row-level security) and CLS (column-level security) rules
- The column names in the rule are lowercase but Snowflake uppercases identifiers
- Both transformations should be applied correctly
Rule configuration:
{
"allowed": [
{"database": "Snowflake", "catalog": "SAMPLE_DATA", "schema": "tpcds_sf10tcl"},
{
"database": "Snowflake",
"table": "customer",
"catalog": "SAMPLE_DATA",
"schema": "tpcds_sf10tcl",
"rls": {"predicate": "C_BIRTH_COUNTRY = 'BRAZIL'"},
"cls": {"c_email_address": "hash"}
}
]
}
"""
sql = "SELECT C_BIRTH_COUNTRY, C_EMAIL_ADDRESS FROM customer LIMIT 10"
statement = SQLStatement(sql, engine="snowflake")
# CLS rules with lowercase column name (as stored in the rule)
cls_rules = {
Table("customer", "tpcds_sf10tcl", "SAMPLE_DATA"): {
"c_email_address": CLSAction.HASH
}
}
# RLS predicate
rls_predicate = statement.parse_predicate("C_BIRTH_COUNTRY = 'BRAZIL'")
rls_predicates = {
Table("customer", "tpcds_sf10tcl", "SAMPLE_DATA"): [rls_predicate]
}
# Schema for qualify() to expand SELECT *
table_schema = {
"SAMPLE_DATA": {
"tpcds_sf10tcl": {
"customer": {
"C_BIRTH_COUNTRY": "VARCHAR",
"C_EMAIL_ADDRESS": "VARCHAR",
}
}
}
}
# Apply CLS first (before RLS)
statement.apply_cls(cls_rules, schema=table_schema)
# Apply RLS after CLS
statement.apply_rls(
"SAMPLE_DATA", "tpcds_sf10tcl", rls_predicates, RLSMethod.AS_SUBQUERY
)
result = statement.format()
# Verify CLS is applied: c_email_address should be hashed with MD5
assert "MD5(TO_CHAR(" in result
assert "C_EMAIL_ADDRESS" in result
# Verify RLS is applied: should have subquery with WHERE clause
assert "WHERE" in result
assert "C_BIRTH_COUNTRY = 'BRAZIL'" in result
# Verify the structure: should be SELECT ... FROM (SELECT * FROM ... WHERE ...) AS ...
assert "SELECT" in result
assert "FROM (" in result
def test_combined_rls_and_cls_case_insensitive_matching() -> None:
"""
Test that RLS and CLS matching is case-insensitive.
Snowflake uppercases identifiers, so the rule column names (lowercase)
must match the query column names (uppercase after qualify()).
"""
sql = "SELECT email FROM users"
statement = SQLStatement(sql, engine="snowflake")
# Rules with lowercase names
cls_rules = {Table("users"): {"email": CLSAction.HASH}}
rls_predicate = statement.parse_predicate("active = TRUE")
rls_predicates = {Table("users"): [rls_predicate]}
table_schema = {"users": {"email": "VARCHAR", "active": "BOOLEAN"}}
# Apply CLS then RLS
statement.apply_cls(cls_rules, schema=table_schema)
statement.apply_rls(None, None, rls_predicates, RLSMethod.AS_SUBQUERY)
result = statement.format()
# Both should be applied despite case differences
assert "MD5" in result # CLS hash applied
assert "active = TRUE" in result # RLS predicate applied