mirror of
https://github.com/apache/superset.git
synced 2026-04-20 00:24:38 +00:00
feat: new Columnar upload form and API (#28192)
This commit is contained in:
Daniel Vaz Gaspar
committed by
GitHub
GitHub
parent
f5843fe588
commit
9a339f08a7
@@ -29,6 +29,7 @@ import { forEach } from 'lodash';
|
||||
|
||||
fetchMock.post('glob:*api/v1/database/1/csv_upload/', {});
|
||||
fetchMock.post('glob:*api/v1/database/1/excel_upload/', {});
|
||||
fetchMock.post('glob:*api/v1/database/1/columnar_upload/', {});
|
||||
|
||||
fetchMock.get(
|
||||
'glob:*api/v1/database/?q=(filters:!((col:allow_file_upload,opr:eq,value:!t)),page:0,page_size:100)',
|
||||
@@ -68,6 +69,13 @@ const excelProps = {
|
||||
type: 'excel',
|
||||
};
|
||||
|
||||
const columnarProps = {
|
||||
show: true,
|
||||
onHide: () => {},
|
||||
allowedExtensions: ['parquet', 'zip'],
|
||||
type: 'columnar',
|
||||
};
|
||||
|
||||
test('CSV, renders the general information elements correctly', () => {
|
||||
render(<UploadDataModal {...csvProps} />, {
|
||||
useRedux: true,
|
||||
@@ -200,6 +208,78 @@ test('Excel, renders the general information elements correctly', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, renders the general information elements correctly', () => {
|
||||
render(<UploadDataModal {...columnarProps} />, {
|
||||
useRedux: true,
|
||||
});
|
||||
|
||||
const cancelButton = screen.getByRole('button', {
|
||||
name: 'Cancel',
|
||||
});
|
||||
const uploadButton = screen.getByRole('button', {
|
||||
name: 'Upload',
|
||||
});
|
||||
const selectButton = screen.getByRole('button', {
|
||||
name: 'Select',
|
||||
});
|
||||
|
||||
const title = screen.getByRole('heading', {
|
||||
name: /columnar upload/i,
|
||||
});
|
||||
const missingTitle = screen.queryByRole('heading', {
|
||||
name: /csv upload/i,
|
||||
});
|
||||
expect(missingTitle).not.toBeInTheDocument();
|
||||
const panel1 = screen.getByRole('heading', {
|
||||
name: /General information/i,
|
||||
});
|
||||
const panel2 = screen.getByRole('heading', {
|
||||
name: /file settings/i,
|
||||
});
|
||||
const panel3 = screen.getByRole('heading', {
|
||||
name: /columns/i,
|
||||
});
|
||||
const panel4 = screen.queryByRole('heading', {
|
||||
name: /rows/i,
|
||||
});
|
||||
expect(panel4).not.toBeInTheDocument();
|
||||
|
||||
const selectDatabase = screen.getByRole('combobox', {
|
||||
name: /select a database/i,
|
||||
});
|
||||
const selectDelimiter = screen.queryByRole('combobox', {
|
||||
name: /choose a delimiter/i,
|
||||
});
|
||||
expect(selectDelimiter).not.toBeInTheDocument();
|
||||
|
||||
const selectSheetName = screen.queryByRole('combobox', {
|
||||
name: /choose sheet name/i,
|
||||
});
|
||||
expect(selectSheetName).not.toBeInTheDocument();
|
||||
const inputTableName = screen.getByRole('textbox', {
|
||||
name: /table name/i,
|
||||
});
|
||||
const inputSchema = screen.getByRole('combobox', {
|
||||
name: /schema/i,
|
||||
});
|
||||
|
||||
const visibleComponents = [
|
||||
cancelButton,
|
||||
uploadButton,
|
||||
selectButton,
|
||||
title,
|
||||
panel1,
|
||||
panel2,
|
||||
panel3,
|
||||
selectDatabase,
|
||||
inputTableName,
|
||||
inputSchema,
|
||||
];
|
||||
visibleComponents.forEach(component => {
|
||||
expect(component).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test('CSV, renders the file settings elements correctly', () => {
|
||||
render(<UploadDataModal {...csvProps} />, {
|
||||
useRedux: true,
|
||||
@@ -282,6 +362,45 @@ test('Excel, renders the file settings elements correctly', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, renders the file settings elements correctly', () => {
|
||||
render(<UploadDataModal {...columnarProps} />, {
|
||||
useRedux: true,
|
||||
});
|
||||
|
||||
expect(screen.queryByText('If Table Already Exists')).not.toBeInTheDocument();
|
||||
const panelHeader = screen.getByRole('heading', {
|
||||
name: /file settings/i,
|
||||
});
|
||||
userEvent.click(panelHeader);
|
||||
const selectTableAlreadyExists = screen.getByRole('combobox', {
|
||||
name: /choose already exists/i,
|
||||
});
|
||||
const inputDecimalCharacter = screen.queryByRole('textbox', {
|
||||
name: /decimal character/i,
|
||||
});
|
||||
expect(inputDecimalCharacter).not.toBeInTheDocument();
|
||||
const selectColumnsDates = screen.queryByRole('combobox', {
|
||||
name: /choose columns to be parsed as dates/i,
|
||||
});
|
||||
expect(selectColumnsDates).not.toBeInTheDocument();
|
||||
const selectNullValues = screen.queryByRole('combobox', {
|
||||
name: /null values/i,
|
||||
});
|
||||
expect(selectNullValues).not.toBeInTheDocument();
|
||||
|
||||
const switchSkipInitialSpace = screen.queryByText('skipInitialSpace');
|
||||
expect(switchSkipInitialSpace).not.toBeInTheDocument();
|
||||
const switchSkipBlankLines = screen.queryByText('skipBlankLines');
|
||||
expect(switchSkipBlankLines).not.toBeInTheDocument();
|
||||
const switchDayFirst = screen.queryByText('dayFirst');
|
||||
expect(switchDayFirst).not.toBeInTheDocument();
|
||||
|
||||
const visibleComponents = [selectTableAlreadyExists];
|
||||
visibleComponents.forEach(component => {
|
||||
expect(component).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test('CSV, renders the columns elements correctly', () => {
|
||||
render(<UploadDataModal {...csvProps} />, {
|
||||
useRedux: true,
|
||||
@@ -291,12 +410,13 @@ test('CSV, renders the columns elements correctly', () => {
|
||||
name: /columns/i,
|
||||
});
|
||||
userEvent.click(panelHeader);
|
||||
const switchDataFrameIndex = screen.getByTestId('dataFrameIndex');
|
||||
userEvent.click(switchDataFrameIndex);
|
||||
const selectIndexColumn = screen.getByRole('combobox', {
|
||||
name: /Choose index column/i,
|
||||
});
|
||||
const switchDataFrameIndex = screen.getByTestId('dataFrameIndex');
|
||||
const inputColumnLabels = screen.getByRole('textbox', {
|
||||
name: /Column labels/i,
|
||||
name: /Index label/i,
|
||||
});
|
||||
const selectColumnsToRead = screen.getByRole('combobox', {
|
||||
name: /Choose columns to read/i,
|
||||
@@ -327,12 +447,13 @@ test('Excel, renders the columns elements correctly', () => {
|
||||
name: /columns/i,
|
||||
});
|
||||
userEvent.click(panelHeader);
|
||||
const switchDataFrameIndex = screen.getByTestId('dataFrameIndex');
|
||||
userEvent.click(switchDataFrameIndex);
|
||||
const selectIndexColumn = screen.getByRole('combobox', {
|
||||
name: /Choose index column/i,
|
||||
});
|
||||
const switchDataFrameIndex = screen.getByTestId('dataFrameIndex');
|
||||
const inputColumnLabels = screen.getByRole('textbox', {
|
||||
name: /Column labels/i,
|
||||
const inputIndexLabel = screen.getByRole('textbox', {
|
||||
name: /Index label/i,
|
||||
});
|
||||
const selectColumnsToRead = screen.getByRole('combobox', {
|
||||
name: /Choose columns to read/i,
|
||||
@@ -348,7 +469,45 @@ test('Excel, renders the columns elements correctly', () => {
|
||||
const visibleComponents = [
|
||||
selectIndexColumn,
|
||||
switchDataFrameIndex,
|
||||
inputColumnLabels,
|
||||
inputIndexLabel,
|
||||
selectColumnsToRead,
|
||||
];
|
||||
visibleComponents.forEach(component => {
|
||||
expect(component).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, renders the columns elements correctly', () => {
|
||||
render(<UploadDataModal {...columnarProps} />, {
|
||||
useRedux: true,
|
||||
});
|
||||
|
||||
const panelHeader = screen.getByRole('heading', {
|
||||
name: /columns/i,
|
||||
});
|
||||
userEvent.click(panelHeader);
|
||||
const selectIndexColumn = screen.queryByRole('combobox', {
|
||||
name: /Choose index column/i,
|
||||
});
|
||||
expect(selectIndexColumn).not.toBeInTheDocument();
|
||||
const switchDataFrameIndex = screen.getByTestId('dataFrameIndex');
|
||||
userEvent.click(switchDataFrameIndex);
|
||||
const inputIndexLabel = screen.getByRole('textbox', {
|
||||
name: /Index label/i,
|
||||
});
|
||||
const selectColumnsToRead = screen.getByRole('combobox', {
|
||||
name: /Choose columns to read/i,
|
||||
});
|
||||
userEvent.click(selectColumnsToRead);
|
||||
|
||||
const columnDataTypes = screen.queryByRole('textbox', {
|
||||
name: /Column data types/i,
|
||||
});
|
||||
expect(columnDataTypes).not.toBeInTheDocument();
|
||||
|
||||
const visibleComponents = [
|
||||
switchDataFrameIndex,
|
||||
inputIndexLabel,
|
||||
selectColumnsToRead,
|
||||
];
|
||||
visibleComponents.forEach(component => {
|
||||
@@ -381,6 +540,17 @@ test('renders the rows elements correctly', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, does not render the rows', () => {
|
||||
render(<UploadDataModal {...columnarProps} />, {
|
||||
useRedux: true,
|
||||
});
|
||||
|
||||
const panelHeader = screen.queryByRole('heading', {
|
||||
name: /rows/i,
|
||||
});
|
||||
expect(panelHeader).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('database and schema are correctly populated', async () => {
|
||||
render(<UploadDataModal {...csvProps} />, {
|
||||
useRedux: true,
|
||||
@@ -546,6 +716,67 @@ test('Excel, form post', async () => {
|
||||
expect(fileData.name).toBe('test.xls');
|
||||
});
|
||||
|
||||
test('Columnar, form post', async () => {
|
||||
render(<UploadDataModal {...columnarProps} />, {
|
||||
useRedux: true,
|
||||
});
|
||||
|
||||
const selectButton = screen.getByRole('button', {
|
||||
name: 'Select',
|
||||
});
|
||||
userEvent.click(selectButton);
|
||||
|
||||
// Select a file from the file dialog
|
||||
const file = new File(['test'], 'test.parquet', { type: 'text' });
|
||||
const inputElement = document.querySelector('input[type="file"]');
|
||||
|
||||
if (inputElement) {
|
||||
userEvent.upload(inputElement, file);
|
||||
}
|
||||
|
||||
const selectDatabase = screen.getByRole('combobox', {
|
||||
name: /select a database/i,
|
||||
});
|
||||
userEvent.click(selectDatabase);
|
||||
await waitFor(() => screen.getByText('database1'));
|
||||
await waitFor(() => screen.getByText('database2'));
|
||||
|
||||
screen.getByText('database1').click();
|
||||
const selectSchema = screen.getByRole('combobox', {
|
||||
name: /schema/i,
|
||||
});
|
||||
userEvent.click(selectSchema);
|
||||
await waitFor(() => screen.getAllByText('public'));
|
||||
screen.getAllByText('public')[1].click();
|
||||
|
||||
// Fill out form fields
|
||||
const inputTableName = screen.getByRole('textbox', {
|
||||
name: /table name/i,
|
||||
});
|
||||
userEvent.type(inputTableName, 'table1');
|
||||
const uploadButton = screen.getByRole('button', {
|
||||
name: 'Upload',
|
||||
});
|
||||
|
||||
userEvent.click(uploadButton);
|
||||
await waitFor(() =>
|
||||
fetchMock.called('glob:*api/v1/database/1/columnar_upload/'),
|
||||
);
|
||||
|
||||
// Get the matching fetch calls made
|
||||
const matchingCalls = fetchMock.calls(
|
||||
'glob:*api/v1/database/1/columnar_upload/',
|
||||
);
|
||||
expect(matchingCalls).toHaveLength(1);
|
||||
const [_, options] = matchingCalls[0];
|
||||
const formData = options?.body as FormData;
|
||||
expect(formData.get('table_name')).toBe('table1');
|
||||
expect(formData.get('schema')).toBe('public');
|
||||
expect(formData.get('table_name')).toBe('table1');
|
||||
const fileData = formData.get('file') as File;
|
||||
expect(fileData.name).toBe('test.parquet');
|
||||
});
|
||||
|
||||
test('CSV, validate file extension returns false', () => {
|
||||
const invalidFileNames = ['out', 'out.exe', 'out.csv.exe', '.csv', 'out.xls'];
|
||||
forEach(invalidFileNames, fileName => {
|
||||
@@ -572,6 +803,25 @@ test('Excel, validate file extension returns false', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, validate file extension returns false', () => {
|
||||
const invalidFileNames = [
|
||||
'out',
|
||||
'out.exe',
|
||||
'out.parquet.exe',
|
||||
'.parquet',
|
||||
'out.excel',
|
||||
];
|
||||
forEach(invalidFileNames, fileName => {
|
||||
const file: UploadFile<any> = {
|
||||
name: fileName,
|
||||
uid: 'xp',
|
||||
size: 100,
|
||||
type: 'text/csv',
|
||||
};
|
||||
expect(validateUploadFileExtension(file, ['parquet', 'zip'])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('CSV, validate file extension returns true', () => {
|
||||
const invalidFileNames = ['out.csv', 'out.tsv', 'out.exe.csv', 'out a.csv'];
|
||||
forEach(invalidFileNames, fileName => {
|
||||
@@ -597,3 +847,21 @@ test('Excel, validate file extension returns true', () => {
|
||||
expect(validateUploadFileExtension(file, ['xls', 'xlsx'])).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('Columnar, validate file extension returns true', () => {
|
||||
const invalidFileNames = [
|
||||
'out.parquet',
|
||||
'out.zip',
|
||||
'out.exe.zip',
|
||||
'out a.parquet',
|
||||
];
|
||||
forEach(invalidFileNames, fileName => {
|
||||
const file: UploadFile<any> = {
|
||||
name: fileName,
|
||||
uid: 'xp',
|
||||
size: 100,
|
||||
type: 'text/csv',
|
||||
};
|
||||
expect(validateUploadFileExtension(file, ['parquet', 'zip'])).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user