QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-06-03 14:49:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: consolidate common code in importers (#11936)

Browse Source 
* chore: consolidate common code in importers

* Small fixes

* Small fixes
This commit is contained in:
Beto Dealmeida
2020-12-10 10:11:58 -08:00
committed by GitHub
parent 38bb6f3f20
commit 9256b6fb3d
25 changed files with 418 additions and 1348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx → superset-frontend/src/components/ImportModal/ImportModal.test.tsx
View File

@@ -22,25 +22,29 @@ import configureStore from 'redux-mock-store';
import { styledMount as mount } from 'spec/helpers/theming';
import { ReactWrapper } from 'enzyme';
import ImportChartModal from 'src/chart/components/ImportModal';
import ImportModelsModal, { StyledIcon } from 'src/components/ImportModal';
import Modal from 'src/common/components/Modal';
const mockStore = configureStore([thunk]);
const store = mockStore({});
const requiredProps = {
resourceName: 'model',
resourceLabel: 'model',
icon: <StyledIcon name="database" />,
passwordsNeededMessage: 'Passwords are needed',
addDangerToast: () => {},
addSuccessToast: () => {},
onChartImport: () => {},
onModelImport: () => {},
show: true,
onHide: () => {},
};
describe('ImportChartModal', () => {
describe('ImportModelsModal', () => {
let wrapper: ReactWrapper;
beforeEach(() => {
wrapper = mount(<ImportChartModal {...requiredProps} />, {
wrapper = mount(<ImportModelsModal {...requiredProps} />, {
context: { store },
});
});
@@ -50,15 +54,15 @@ describe('ImportChartModal', () => {
});
it('renders', () => {
expect(wrapper.find(ImportChartModal)).toExist();
expect(wrapper.find(ImportModelsModal)).toExist();
});
it('renders a Modal', () => {
expect(wrapper.find(Modal)).toExist();
});
it('renders "Import Chart" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import Chart');
it('renders "Import model" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import model');
});
it('renders a label and a file input field', () => {
@@ -67,7 +71,7 @@ describe('ImportChartModal', () => {
});
it('should attach the label to the input field', () => {
const id = 'chartFile';
const id = 'modelFile';
expect(wrapper.find('label').prop('htmlFor')).toBe(id);
expect(wrapper.find('input').prop('id')).toBe(id);
});
@@ -83,7 +87,7 @@ describe('ImportChartModal', () => {
});
it('should render the import button enabled when a file is selected', () => {
const file = new File([new ArrayBuffer(1)], 'chart_export.zip');
const file = new File([new ArrayBuffer(1)], 'model_export.zip');
wrapper.find('input').simulate('change', { target: { files: [file] } });
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
@@ -93,7 +97,7 @@ describe('ImportChartModal', () => {
it('should render password fields when needed for import', () => {
const wrapperWithPasswords = mount(
<ImportChartModal
<ImportModelsModal
{...requiredProps}
passwordFields={['databases/examples.yaml']}
/>,

135
superset-frontend/src/chart/components/ImportModal/index.tsx → superset-frontend/src/components/ImportModal/index.tsx
View File

@@ -17,30 +17,107 @@
* under the License.
*/
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { t } from '@superset-ui/core';
import { styled, t } from '@superset-ui/core';
import Icon from 'src//components/Icon';
import Modal from 'src/common/components/Modal';
import {
StyledIcon,
StyledInputContainer,
} from 'src/views/CRUD/data/database/DatabaseModal';
import { useImportResource } from 'src/views/CRUD/hooks';
import { ChartObject } from 'src/views/CRUD/chart/types';
export interface ImportChartModalProps {
export const StyledIcon = styled(Icon)`
margin: auto ${({ theme }) => theme.gridUnit * 2}px auto 0;
`;
const StyledInputContainer = styled.div`
margin-bottom: ${({ theme }) => theme.gridUnit * 2}px;
&.extra-container {
padding-top: 8px;
}
.helper {
display: block;
padding: ${({ theme }) => theme.gridUnit}px 0;
color: ${({ theme }) => theme.colors.grayscale.base};
font-size: ${({ theme }) => theme.typography.sizes.s - 1}px;
text-align: left;
.required {
margin-left: ${({ theme }) => theme.gridUnit / 2}px;
color: ${({ theme }) => theme.colors.error.base};
}
}
.input-container {
display: flex;
align-items: center;
label {
display: flex;
margin-right: ${({ theme }) => theme.gridUnit * 2}px;
}
i {
margin: 0 ${({ theme }) => theme.gridUnit}px;
}
}
input,
textarea {
flex: 1 1 auto;
}
textarea {
height: 160px;
resize: none;
}
input::placeholder,
textarea::placeholder {
color: ${({ theme }) => theme.colors.grayscale.light1};
}
textarea,
input[type='text'],
input[type='number'] {
padding: ${({ theme }) => theme.gridUnit * 1.5}px
${({ theme }) => theme.gridUnit * 2}px;
border-style: none;
border: 1px solid ${({ theme }) => theme.colors.grayscale.light2};
border-radius: ${({ theme }) => theme.gridUnit}px;
&[name='name'] {
flex: 0 1 auto;
width: 40%;
}
&[name='sqlalchemy_uri'] {
margin-right: ${({ theme }) => theme.gridUnit * 3}px;
}
}
`;
export interface ImportModelsModalProps {
resourceName: string;
resourceLabel: string;
icon: React.ReactNode;
passwordsNeededMessage: string;
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
onChartImport: () => void;
onModelImport: () => void;
show: boolean;
onHide: () => void;
passwordFields?: string[];
setPasswordFields?: (passwordFields: string[]) => void;
}
const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
const ImportModelsModal: FunctionComponent<ImportModelsModalProps> = ({
resourceName,
resourceLabel,
icon,
passwordsNeededMessage,
addDangerToast,
addSuccessToast,
onChartImport,
onModelImport,
show,
onHide,
passwordFields = [],
@@ -68,11 +145,11 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
const {
state: { passwordsNeeded },
importResource,
} = useImportResource<ChartObject>('chart', t('chart'), handleErrorMsg);
} = useImportResource<any>(resourceName, resourceLabel, handleErrorMsg);
useEffect(() => {
setPasswordFields(passwordsNeeded);
}, [passwordsNeeded]);
}, [passwordsNeeded, setPasswordFields]);
// Functions
const hide = () => {
@@ -87,9 +164,9 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
importResource(uploadFile, passwords).then(result => {
if (result) {
addSuccessToast(t('The charts have been imported'));
addSuccessToast(t('The import was successful'));
clearModal();
onChartImport();
onModelImport();
}
});
};
@@ -108,15 +185,7 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
<>
<h5>Database passwords</h5>
<StyledInputContainer>
<div className="helper">
{t(
'The passwords for the databases below are needed in order to ' +
'import them together with the charts. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
</div>
<div className="helper">{passwordsNeededMessage}</div>
</StyledInputContainer>
{passwordFields.map(fileName => (
<StyledInputContainer key={`password-for-${fileName}`}>
@@ -126,7 +195,7 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
</div>
<input
name={`password-${fileName}`}
autoComplete="off"
autoComplete={`password-${fileName}`}
type="password"
value={passwords[fileName]}
onChange={event =>
@@ -146,8 +215,8 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
return (
<Modal
name="chart"
className="chart-modal"
name="model"
className="import-model-modal"
disablePrimaryButton={uploadFile === null}
onHandledPrimaryAction={onUpload}
onHide={hide}
@@ -156,23 +225,23 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
show={show}
title={
<h4>
<StyledIcon name="nav-charts" />
{t('Import Chart')}
{icon}
{t('Import %s', resourceLabel)}
</h4>
}
>
<StyledInputContainer>
<div className="control-label">
<label htmlFor="chartFile">
<label htmlFor="modelFile">
{t('File')}
<span className="required">*</span>
</label>
</div>
<input
ref={fileInputRef}
data-test="chart-file-input"
name="chartFile"
id="chartFile"
data-test="model-file-input"
name="modelFile"
id="modelFile"
type="file"
accept=".yaml,.json,.yml,.zip"
onChange={changeFile}
@@ -183,4 +252,4 @@ const ImportChartModal: FunctionComponent<ImportChartModalProps> = ({
);
};
export default ImportChartModal;
export default ImportModelsModal;

106
superset-frontend/src/dashboard/components/ImportModal/ImportModal.test.tsx
View File

@@ -1,106 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import thunk from 'redux-thunk';
import configureStore from 'redux-mock-store';
import { styledMount as mount } from 'spec/helpers/theming';
import { ReactWrapper } from 'enzyme';
import ImportDashboardModal from 'src/dashboard/components/ImportModal';
import Modal from 'src/common/components/Modal';
const mockStore = configureStore([thunk]);
const store = mockStore({});
const requiredProps = {
addDangerToast: () => {},
addSuccessToast: () => {},
onDashboardImport: () => {},
show: true,
onHide: () => {},
};
describe('ImportDashboardModal', () => {
let wrapper: ReactWrapper;
beforeEach(() => {
wrapper = mount(<ImportDashboardModal {...requiredProps} />, {
context: { store },
});
});
afterEach(() => {
jest.clearAllMocks();
});
it('renders', () => {
expect(wrapper.find(ImportDashboardModal)).toExist();
});
it('renders a Modal', () => {
expect(wrapper.find(Modal)).toExist();
});
it('renders "Import Dashboard" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import Dashboard');
});
it('renders a label and a file input field', () => {
expect(wrapper.find('input[type="file"]')).toExist();
expect(wrapper.find('label')).toExist();
});
it('should attach the label to the input field', () => {
const id = 'dashboardFile';
expect(wrapper.find('label').prop('htmlFor')).toBe(id);
expect(wrapper.find('input').prop('id')).toBe(id);
});
it('should render the close, import and cancel buttons', () => {
expect(wrapper.find('button')).toHaveLength(3);
});
it('should render the import button initially disabled', () => {
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
true,
);
});
it('should render the import button enabled when a file is selected', () => {
const file = new File([new ArrayBuffer(1)], 'dashboard_export.zip');
wrapper.find('input').simulate('change', { target: { files: [file] } });
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
false,
);
});
it('should render password fields when needed for import', () => {
const wrapperWithPasswords = mount(
<ImportDashboardModal
{...requiredProps}
passwordFields={['databases/examples.yaml']}
/>,
{
context: { store },
},
);
expect(wrapperWithPasswords.find('input[type="password"]')).toExist();
});
});

190
superset-frontend/src/dashboard/components/ImportModal/index.tsx
View File

@@ -1,190 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { t } from '@superset-ui/core';
import Modal from 'src/common/components/Modal';
import {
StyledIcon,
StyledInputContainer,
} from 'src/views/CRUD/data/database/DatabaseModal';
import { useImportResource } from 'src/views/CRUD/hooks';
import { DashboardObject } from 'src/views/CRUD/dashboard/types';
export interface ImportDashboardModalProps {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
onDashboardImport: () => void;
show: boolean;
onHide: () => void;
passwordFields?: string[];
setPasswordFields?: (passwordFields: string[]) => void;
}
const ImportDashboardModal: FunctionComponent<ImportDashboardModalProps> = ({
addDangerToast,
addSuccessToast,
onDashboardImport,
show,
onHide,
passwordFields = [],
setPasswordFields = () => {},
}) => {
const [uploadFile, setUploadFile] = useState<File | null>(null);
const [isHidden, setIsHidden] = useState<boolean>(true);
const [passwords, setPasswords] = useState<Record<string, string>>({});
const fileInputRef = useRef<HTMLInputElement>(null);
const clearModal = () => {
setUploadFile(null);
setPasswordFields([]);
setPasswords({});
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.value = '';
}
};
const handleErrorMsg = (msg: string) => {
clearModal();
addDangerToast(msg);
};
const {
state: { passwordsNeeded },
importResource,
} = useImportResource<DashboardObject>(
'dashboard',
t('dashboard'),
handleErrorMsg,
);
useEffect(() => {
setPasswordFields(passwordsNeeded);
}, [passwordsNeeded]);
// Functions
const hide = () => {
setIsHidden(true);
onHide();
};
const onUpload = () => {
if (uploadFile === null) {
return;
}
importResource(uploadFile, passwords).then(result => {
if (result) {
addSuccessToast(t('The dashboards have been imported'));
clearModal();
onDashboardImport();
}
});
};
const changeFile = (event: React.ChangeEvent<HTMLInputElement>) => {
const { files } = event.target as HTMLInputElement;
setUploadFile((files && files[0]) || null);
};
const renderPasswordFields = () => {
if (passwordFields.length === 0) {
return null;
}
return (
<>
<h5>Database passwords</h5>
<StyledInputContainer>
<div className="helper">
{t(
'The passwords for the databases below are needed in order to ' +
'import them together with the dashboards. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
</div>
</StyledInputContainer>
{passwordFields.map(fileName => (
<StyledInputContainer key={`password-for-${fileName}`}>
<div className="control-label">
{fileName}
<span className="required">*</span>
</div>
<input
name={`password-${fileName}`}
autoComplete="off"
type="password"
value={passwords[fileName]}
onChange={event =>
setPasswords({ ...passwords, [fileName]: event.target.value })
}
/>
</StyledInputContainer>
))}
</>
);
};
// Show/hide
if (isHidden && show) {
setIsHidden(false);
}
return (
<Modal
name="dashboard"
className="dashboard-modal"
disablePrimaryButton={uploadFile === null}
onHandledPrimaryAction={onUpload}
onHide={hide}
primaryButtonName={t('Import')}
width="750px"
show={show}
title={
<h4>
<StyledIcon name="nav-dashboard" />
{t('Import Dashboard')}
</h4>
}
>
<StyledInputContainer>
<div className="control-label">
<label htmlFor="dashboardFile">
{t('File')}
<span className="required">*</span>
</label>
</div>
<input
ref={fileInputRef}
data-test="dashboard-file-input"
name="dashboardFile"
id="dashboardFile"
type="file"
accept=".yaml,.json,.yml,.zip"
onChange={changeFile}
/>
</StyledInputContainer>
{renderPasswordFields()}
</Modal>
);
};
export default ImportDashboardModal;

96
superset-frontend/src/database/components/ImportModal/ImportModal.test.tsx
View File

@@ -1,96 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import { styledMount as mount } from 'spec/helpers/theming';
import { ReactWrapper } from 'enzyme';
import ImportDatabaseModal from 'src/database/components/ImportModal';
import Modal from 'src/common/components/Modal';
const requiredProps = {
addDangerToast: () => {},
addSuccessToast: () => {},
onDatabaseImport: () => {},
show: true,
onHide: () => {},
};
describe('ImportDatabaseModal', () => {
let wrapper: ReactWrapper;
beforeEach(() => {
wrapper = mount(<ImportDatabaseModal {...requiredProps} />);
});
afterEach(() => {
jest.clearAllMocks();
});
it('renders', () => {
expect(wrapper.find(ImportDatabaseModal)).toExist();
});
it('renders a Modal', () => {
expect(wrapper.find(Modal)).toExist();
});
it('renders "Import Database" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import Database');
});
it('renders a label and a file input field', () => {
expect(wrapper.find('input[type="file"]')).toExist();
expect(wrapper.find('label')).toExist();
});
it('should attach the label to the input field', () => {
const id = 'databaseFile';
expect(wrapper.find('label').prop('htmlFor')).toBe(id);
expect(wrapper.find('input').prop('id')).toBe(id);
});
it('should render the close, import and cancel buttons', () => {
expect(wrapper.find('button')).toHaveLength(3);
});
it('should render the import button initially disabled', () => {
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
true,
);
});
it('should render the import button enabled when a file is selected', () => {
const file = new File([new ArrayBuffer(1)], 'database_export.zip');
wrapper.find('input').simulate('change', { target: { files: [file] } });
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
false,
);
});
it('should render password fields when needed for import', () => {
const wrapperWithPasswords = mount(
<ImportDatabaseModal
{...requiredProps}
passwordFields={['databases/examples.yaml']}
/>,
);
expect(wrapperWithPasswords.find('input[type="password"]')).toExist();
});
});

191
superset-frontend/src/database/components/ImportModal/index.tsx
View File

@@ -1,191 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { t } from '@superset-ui/core';
import Modal from 'src/common/components/Modal';
import {
StyledIcon,
StyledInputContainer,
} from 'src/views/CRUD/data/database/DatabaseModal';
import { useImportResource } from 'src/views/CRUD/hooks';
import { DatabaseObject } from 'src/views/CRUD/data/database/types';
export interface ImportDatabaseModalProps {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
onDatabaseImport: () => void;
show: boolean;
onHide: () => void;
passwordFields?: string[];
setPasswordFields?: (passwordFields: string[]) => void;
}
const ImportDatabaseModal: FunctionComponent<ImportDatabaseModalProps> = ({
addDangerToast,
addSuccessToast,
onDatabaseImport,
show,
onHide,
passwordFields = [],
setPasswordFields = () => {},
}) => {
const [uploadFile, setUploadFile] = useState<File | null>(null);
const [isHidden, setIsHidden] = useState<boolean>(true);
const [passwords, setPasswords] = useState<Record<string, string>>({});
const fileInputRef = useRef<HTMLInputElement>(null);
const clearModal = () => {
setUploadFile(null);
setPasswords({});
setPasswordFields([]);
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.value = '';
}
};
const handleErrorMsg = (msg: string) => {
clearModal();
addDangerToast(msg);
};
const {
state: { passwordsNeeded },
importResource,
} = useImportResource<DatabaseObject>(
'database',
t('database'),
handleErrorMsg,
);
useEffect(() => {
setPasswordFields(passwordsNeeded);
}, [passwordsNeeded]);
// Functions
const hide = () => {
setIsHidden(true);
onHide();
};
const onUpload = () => {
if (uploadFile === null) {
return;
}
importResource(uploadFile, passwords).then(result => {
if (result) {
addSuccessToast(t('The databases have been imported'));
clearModal();
onDatabaseImport();
}
});
};
const changeFile = (event: React.ChangeEvent<HTMLInputElement>) => {
const { files } = event.target as HTMLInputElement;
setUploadFile((files && files[0]) || null);
};
const renderPasswordFields = () => {
if (passwordFields.length === 0) {
return null;
}
return (
<>
<h5>Passwords</h5>
<StyledInputContainer>
<div className="helper">
{t('Please provide the password for the databases below')}
</div>
</StyledInputContainer>
{passwordFields.map(fileName => (
<StyledInputContainer key={`password-for-${fileName}`}>
<div className="control-label">
{fileName}
<span className="required">*</span>
</div>
<input
name={`password-${fileName}`}
autoComplete="off"
type="password"
value={passwords[fileName]}
onChange={event =>
setPasswords({ ...passwords, [fileName]: event.target.value })
}
/>
</StyledInputContainer>
))}
</>
);
};
// Show/hide
if (isHidden && show) {
setIsHidden(false);
}
return (
<Modal
name="database"
className="database-modal"
disablePrimaryButton={uploadFile === null}
onHandledPrimaryAction={onUpload}
onHide={hide}
primaryButtonName={t('Import')}
width="750px"
show={show}
title={
<h4>
<StyledIcon name="database" />
{t('Import Database')}
</h4>
}
>
<StyledInputContainer>
<div className="control-label">
<label htmlFor="databaseFile">
{t('File')}
<span className="required">*</span>
</label>
</div>
<input
ref={fileInputRef}
data-test="database-file-input"
name="databaseFile"
id="databaseFile"
type="file"
accept=".yaml,.json,.yml,.zip"
onChange={changeFile}
/>
<div className="helper">
{t(
'Please note that the "Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
</div>
</StyledInputContainer>
{renderPasswordFields()}
</Modal>
);
};
export default ImportDatabaseModal;

106
superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx
View File

@@ -1,106 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import thunk from 'redux-thunk';
import configureStore from 'redux-mock-store';
import { styledMount as mount } from 'spec/helpers/theming';
import { ReactWrapper } from 'enzyme';
import ImportDatasetModal from 'src/datasource/components/ImportModal';
import Modal from 'src/common/components/Modal';
const mockStore = configureStore([thunk]);
const store = mockStore({});
const requiredProps = {
addDangerToast: () => {},
addSuccessToast: () => {},
onDatasetImport: () => {},
show: true,
onHide: () => {},
};
describe('ImportDatasetModal', () => {
let wrapper: ReactWrapper;
beforeEach(() => {
wrapper = mount(<ImportDatasetModal {...requiredProps} />, {
context: { store },
});
});
afterEach(() => {
jest.clearAllMocks();
});
it('renders', () => {
expect(wrapper.find(ImportDatasetModal)).toExist();
});
it('renders a Modal', () => {
expect(wrapper.find(Modal)).toExist();
});
it('renders "Import Dataset" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import Dataset');
});
it('renders a label and a file input field', () => {
expect(wrapper.find('input[type="file"]')).toExist();
expect(wrapper.find('label')).toExist();
});
it('should attach the label to the input field', () => {
const id = 'datasetFile';
expect(wrapper.find('label').prop('htmlFor')).toBe(id);
expect(wrapper.find('input').prop('id')).toBe(id);
});
it('should render the close, import and cancel buttons', () => {
expect(wrapper.find('button')).toHaveLength(3);
});
it('should render the import button initially disabled', () => {
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
true,
);
});
it('should render the import button enabled when a file is selected', () => {
const file = new File([new ArrayBuffer(1)], 'dataset_export.zip');
wrapper.find('input').simulate('change', { target: { files: [file] } });
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
false,
);
});
it('should render password fields when needed for import', () => {
const wrapperWithPasswords = mount(
<ImportDatasetModal
{...requiredProps}
passwordFields={['databases/examples.yaml']}
/>,
{
context: { store },
},
);
expect(wrapperWithPasswords.find('input[type="password"]')).toExist();
});
});

186
superset-frontend/src/datasource/components/ImportModal/index.tsx
View File

@@ -1,186 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { t } from '@superset-ui/core';
import Modal from 'src/common/components/Modal';
import {
StyledIcon,
StyledInputContainer,
} from 'src/views/CRUD/data/database/DatabaseModal';
import { useImportResource } from 'src/views/CRUD/hooks';
import { DatasetObject } from 'src/views/CRUD/data/dataset/types';
export interface ImportDatasetModalProps {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
onDatasetImport: () => void;
show: boolean;
onHide: () => void;
passwordFields?: string[];
setPasswordFields?: (passwordFields: string[]) => void;
}
const ImportDatasetModal: FunctionComponent<ImportDatasetModalProps> = ({
addDangerToast,
addSuccessToast,
onDatasetImport,
show,
onHide,
passwordFields = [],
setPasswordFields = () => {},
}) => {
const [uploadFile, setUploadFile] = useState<File | null>(null);
const [isHidden, setIsHidden] = useState<boolean>(true);
const [passwords, setPasswords] = useState<Record<string, string>>({});
const fileInputRef = useRef<HTMLInputElement>(null);
const clearModal = () => {
setUploadFile(null);
setPasswordFields([]);
setPasswords({});
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.value = '';
}
};
const handleErrorMsg = (msg: string) => {
clearModal();
addDangerToast(msg);
};
const {
state: { passwordsNeeded },
importResource,
} = useImportResource<DatasetObject>('dataset', t('dataset'), handleErrorMsg);
useEffect(() => {
setPasswordFields(passwordsNeeded);
}, [passwordsNeeded]);
// Functions
const hide = () => {
setIsHidden(true);
onHide();
};
const onUpload = () => {
if (uploadFile === null) {
return;
}
importResource(uploadFile, passwords).then(result => {
if (result) {
addSuccessToast(t('The datasets have been imported'));
clearModal();
onDatasetImport();
}
});
};
const changeFile = (event: React.ChangeEvent<HTMLInputElement>) => {
const { files } = event.target as HTMLInputElement;
setUploadFile((files && files[0]) || null);
};
const renderPasswordFields = () => {
if (passwordFields.length === 0) {
return null;
}
return (
<>
<h5>Database passwords</h5>
<StyledInputContainer>
<div className="helper">
{t(
'The passwords for the databases below are needed in order to ' +
'import them together with the datasets. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
</div>
</StyledInputContainer>
{passwordFields.map(fileName => (
<StyledInputContainer key={`password-for-${fileName}`}>
<div className="control-label">
{fileName}
<span className="required">*</span>
</div>
<input
name={`password-${fileName}`}
autoComplete="off"
type="password"
value={passwords[fileName]}
onChange={event =>
setPasswords({ ...passwords, [fileName]: event.target.value })
}
/>
</StyledInputContainer>
))}
</>
);
};
// Show/hide
if (isHidden && show) {
setIsHidden(false);
}
return (
<Modal
name="dataset"
className="dataset-modal"
disablePrimaryButton={uploadFile === null}
onHandledPrimaryAction={onUpload}
onHide={hide}
primaryButtonName={t('Import')}
width="750px"
show={show}
title={
<h4>
<StyledIcon name="table" />
{t('Import Dataset')}
</h4>
}
>
<StyledInputContainer>
<div className="control-label">
<label htmlFor="datasetFile">
{t('File')}
<span className="required">*</span>
</label>
</div>
<input
ref={fileInputRef}
data-test="dataset-file-input"
name="datasetFile"
id="datasetFile"
type="file"
accept=".yaml,.json,.yml,.zip"
onChange={changeFile}
/>
</StyledInputContainer>
{renderPasswordFields()}
</Modal>
);
};
export default ImportDatasetModal;

30
superset-frontend/src/views/CRUD/chart/ChartList.tsx
View File

@@ -43,7 +43,9 @@ import ListView, {
} from 'src/components/ListView';
import withToasts from 'src/messageToasts/enhancers/withToasts';
import PropertiesModal from 'src/explore/components/PropertiesModal';
import ImportChartModal from 'src/chart/components/ImportModal/index';
import ImportModelsModal, {
StyledIcon,
} from 'src/components/ImportModal/index';
import Chart from 'src/types/Chart';
import TooltipWrapper from 'src/components/TooltipWrapper';
import ChartCard from './ChartCard';
@@ -130,13 +132,13 @@ function ChartList(props: ChartListProps) {
const [importingChart, showImportModal] = useState<boolean>(false);
const [passwordFields, setPasswordFields] = useState<string[]>([]);
function openChartImportModal() {
const openChartImportModal = () => {
showImportModal(true);
}
};
function closeChartImportModal() {
const closeChartImportModal = () => {
showImportModal(false);
}
};
const handleChartImport = () => {
showImportModal(false);
@@ -568,12 +570,22 @@ function ChartList(props: ChartListProps) {
}}
</ConfirmStatusChange>
<ImportChartModal
show={importingChart}
onHide={closeChartImportModal}
<ImportModelsModal
resourceName="chart"
resourceLabel={t('chart')}
icon={<StyledIcon name="nav-charts" />}
passwordsNeededMessage={t(
'The passwords for the databases below are needed in order to ' +
'import them together with the charts. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
onChartImport={handleChartImport}
onModelImport={handleChartImport}
show={importingChart}
onHide={closeChartImportModal}
passwordFields={passwordFields}
setPasswordFields={setPasswordFields}
/>

56
superset-frontend/src/views/CRUD/dashboard/DashboardList.tsx
View File

@@ -37,7 +37,9 @@ import Icon from 'src/components/Icon';
import FaveStar from 'src/components/FaveStar';
import PropertiesModal from 'src/dashboard/components/PropertiesModal';
import TooltipWrapper from 'src/components/TooltipWrapper';
import ImportDashboardModal from 'src/dashboard/components/ImportModal/index';
import ImportModelsModal, {
StyledIcon,
} from 'src/components/ImportModal/index';
import Dashboard from 'src/dashboard/containers/Dashboard';
import DashboardCard from './DashboardCard';
@@ -67,6 +69,8 @@ interface Dashboard {
}
function DashboardList(props: DashboardListProps) {
const { addDangerToast, addSuccessToast } = props;
const {
state: {
loading,
@@ -82,13 +86,13 @@ function DashboardList(props: DashboardListProps) {
} = useListViewResource<Dashboard>(
'dashboard',
t('dashboard'),
props.addDangerToast,
addDangerToast,
);
const dashboardIds = useMemo(() => dashboards.map(d => d.id), [dashboards]);
const [saveFavoriteStatus, favoriteStatus] = useFavoriteStatus(
'dashboard',
dashboardIds,
props.addDangerToast,
addDangerToast,
);
const [dashboardToEdit, setDashboardToEdit] = useState<Dashboard | null>(
null,
@@ -97,13 +101,13 @@ function DashboardList(props: DashboardListProps) {
const [importingDashboard, showImportModal] = useState<boolean>(false);
const [passwordFields, setPasswordFields] = useState<string[]>([]);
function openDashboardImportModal() {
const openDashboardImportModal = () => {
showImportModal(true);
}
};
function closeDashboardImportModal() {
const closeDashboardImportModal = () => {
showImportModal(false);
}
};
const handleDashboardImport = () => {
showImportModal(false);
@@ -136,7 +140,7 @@ function DashboardList(props: DashboardListProps) {
);
},
createErrorHandler(errMsg =>
props.addDangerToast(
addDangerToast(
t('An error occurred while fetching dashboards: %s', errMsg),
),
),
@@ -150,10 +154,10 @@ function DashboardList(props: DashboardListProps) {
)}`,
}).then(
({ json = {} }) => {
props.addSuccessToast(json.message);
addSuccessToast(json.message);
},
createErrorHandler(errMsg =>
props.addDangerToast(
addDangerToast(
t('There was an issue deleting the selected dashboards: ', errMsg),
),
),
@@ -251,8 +255,8 @@ function DashboardList(props: DashboardListProps) {
handleDashboardDelete(
original,
refreshData,
props.addSuccessToast,
props.addDangerToast,
addSuccessToast,
addDangerToast,
);
const handleEdit = () => openDashboardEditModal(original);
const handleExport = () => handleBulkDashboardExport([original]);
@@ -346,7 +350,7 @@ function DashboardList(props: DashboardListProps) {
'dashboard',
'owners',
createErrorHandler(errMsg =>
props.addDangerToast(
addDangerToast(
t(
'An error occurred while fetching dashboard owner values: %s',
errMsg,
@@ -367,7 +371,7 @@ function DashboardList(props: DashboardListProps) {
'dashboard',
'created_by',
createErrorHandler(errMsg =>
props.addDangerToast(
addDangerToast(
t(
'An error occurred while fetching dashboard created by values: %s',
errMsg,
@@ -426,8 +430,8 @@ function DashboardList(props: DashboardListProps) {
bulkSelectEnabled={bulkSelectEnabled}
refreshData={refreshData}
loading={loading}
addDangerToast={props.addDangerToast}
addSuccessToast={props.addSuccessToast}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
openDashboardEditModal={openDashboardEditModal}
saveFavoriteStatus={saveFavoriteStatus}
favoriteStatus={favoriteStatus[dashboard.id]}
@@ -526,16 +530,26 @@ function DashboardList(props: DashboardListProps) {
);
}}
</ConfirmStatusChange>
<ImportDashboardModal
<ImportModelsModal
resourceName="dashboard"
resourceLabel={t('dashboard')}
icon={<StyledIcon name="nav-dashboard" />}
passwordsNeededMessage={t(
'The passwords for the databases below are needed in order to ' +
'import them together with the dashboards. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
onModelImport={handleDashboardImport}
show={importingDashboard}
onHide={closeDashboardImportModal}
addDangerToast={props.addDangerToast}
addSuccessToast={props.addSuccessToast}
onDashboardImport={handleDashboardImport}
passwordFields={passwordFields}
setPasswordFields={setPasswordFields}
/>
:
</>
);
}

21
superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx
View File

@@ -29,7 +29,9 @@ import TooltipWrapper from 'src/components/TooltipWrapper';
import Icon from 'src/components/Icon';
import ListView, { Filters } from 'src/components/ListView';
import { commonMenuData } from 'src/views/CRUD/data/common';
import ImportDatabaseModal from 'src/database/components/ImportModal/index';
import ImportModelsModal, {
StyledIcon,
} from 'src/components/ImportModal/index';
import DatabaseModal from './DatabaseModal';
import { DatabaseObject } from './types';
@@ -425,12 +427,21 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) {
pageSize={PAGE_SIZE}
/>
<ImportDatabaseModal
show={importingDatabase}
onHide={closeDatabaseImportModal}
<ImportModelsModal
resourceName="database"
resourceLabel={t('database')}
icon={<StyledIcon name="database" />}
passwordsNeededMessage={t(
'The passwords for the databases below are needed in order to ' +
'import them. Please note that the "Secure Extra" and "Certificate" ' +
'sections of the database configuration are not present in export ' +
'files, and should be added manually after the import if they are needed.',
)}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
onDatabaseImport={handleDatabaseImport}
onModelImport={handleDatabaseImport}
show={importingDatabase}
onHide={closeDatabaseImportModal}
passwordFields={passwordFields}
setPasswordFields={setPasswordFields}
/>

4
superset-frontend/src/views/CRUD/data/database/DatabaseModal.tsx
View File

@@ -39,11 +39,11 @@ interface DatabaseModalProps {
database?: DatabaseObject | null; // If included, will go into edit mode
}
export const StyledIcon = styled(Icon)`
const StyledIcon = styled(Icon)`
margin: auto ${({ theme }) => theme.gridUnit * 2}px auto 0;
`;
export const StyledInputContainer = styled.div`
const StyledInputContainer = styled.div`
margin-bottom: ${({ theme }) => theme.gridUnit * 2}px;
&.extra-container {

22
superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx
View File

@@ -45,7 +45,9 @@ import TooltipWrapper from 'src/components/TooltipWrapper';
import Icon from 'src/components/Icon';
import FacePile from 'src/components/FacePile';
import CertifiedIconWithTooltip from 'src/components/CertifiedIconWithTooltip';
import ImportDatasetModal from 'src/datasource/components/ImportModal/index';
import ImportModelsModal, {
StyledIcon,
} from 'src/components/ImportModal/index';
import { isFeatureEnabled, FeatureFlag } from 'src/featureFlags';
import AddDatasetModal from './AddDatasetModal';
@@ -647,12 +649,22 @@ const DatasetList: FunctionComponent<DatasetListProps> = ({
}}
</ConfirmStatusChange>
<ImportDatasetModal
show={importingDataset}
onHide={closeDatasetImportModal}
<ImportModelsModal
resourceName="dataset"
resourceLabel={t('dataset')}
icon={<StyledIcon name="table" />}
passwordsNeededMessage={t(
'The passwords for the databases below are needed in order to ' +
'import them together with the datasets. Please note that the ' +
'"Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
onDatasetImport={handleDatasetImport}
onModelImport={handleDatasetImport}
show={importingDataset}
onHide={closeDatasetImportModal}
passwordFields={passwordFields}
setPasswordFields={setPasswordFields}
/>

19
superset/app.py
View File

@@ -315,15 +315,16 @@ class SupersetAppInitializer:
#
# Add links
#
appbuilder.add_link(
"Import Dashboards",
label=__("Import Dashboards"),
href="/superset/import_dashboards",
icon="fa-cloud-upload",
category="Manage",
category_label=__("Manage"),
category_icon="fa-wrench",
)
if not feature_flag_manager.is_feature_enabled("VERSIONED_EXPORT"):
appbuilder.add_link(
"Import Dashboards",
label=__("Import Dashboards"),
href="/superset/import_dashboards",
icon="fa-cloud-upload",
category="Manage",
category_label=__("Manage"),
category_icon="fa-wrench",
)
appbuilder.add_link(
"SQL Editor",
label=_("SQL Editor"),

2
superset/charts/commands/export.py
View File

@@ -43,7 +43,7 @@ class ExportChartsCommand(ExportModelsCommand):
not_found = ChartNotFoundError
@staticmethod
def export(model: Slice) -> Iterator[Tuple[str, str]]:
def _export(model: Slice) -> Iterator[Tuple[str, str]]:
chart_slug = secure_filename(model.slice_name)
file_name = f"charts/{chart_slug}.yaml"

114
superset/charts/commands/importers/v1/__init__.py
View File

@@ -15,70 +15,59 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, Dict, List, Optional, Set
from typing import Any, Dict, Set
from marshmallow import Schema, validate
from marshmallow.exceptions import ValidationError
from marshmallow import Schema
from sqlalchemy.orm import Session
from superset import db
from superset.charts.commands.exceptions import ChartImportError
from superset.charts.commands.importers.v1.utils import import_chart
from superset.charts.dao import ChartDAO
from superset.charts.schemas import ImportV1ChartSchema
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import (
load_metadata,
load_yaml,
METADATA_FILE_NAME,
)
from superset.commands.importers.v1 import ImportModelsCommand
from superset.databases.commands.importers.v1.utils import import_database
from superset.databases.schemas import ImportV1DatabaseSchema
from superset.datasets.commands.importers.v1.utils import import_dataset
from superset.datasets.schemas import ImportV1DatasetSchema
from superset.models.core import Database
from superset.models.slice import Slice
schemas: Dict[str, Schema] = {
"charts/": ImportV1ChartSchema(),
"datasets/": ImportV1DatasetSchema(),
"databases/": ImportV1DatabaseSchema(),
}
class ImportChartsCommand(BaseCommand):
class ImportChartsCommand(ImportModelsCommand):
"""Import charts"""
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
dao = ChartDAO
model_name = "chart"
schemas: Dict[str, Schema] = {
"charts/": ImportV1ChartSchema(),
"datasets/": ImportV1DatasetSchema(),
"databases/": ImportV1DatabaseSchema(),
}
import_error = ChartImportError
def _import_bundle(self, session: Session) -> None:
@staticmethod
def _import(session: Session, configs: Dict[str, Any]) -> None:
# discover datasets associated with charts
dataset_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("charts/"):
dataset_uuids.add(config["dataset_uuid"])
# discover databases associated with datasets
database_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("datasets/") and config["uuid"] in dataset_uuids:
database_uuids.add(config["database_uuid"])
# import related databases
database_ids: Dict[str, int] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("databases/") and config["uuid"] in database_uuids:
database = import_database(session, config, overwrite=False)
database_ids[str(database.uuid)] = database.id
# import datasets with the correct parent ref
dataset_info: Dict[str, Dict[str, Any]] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("datasets/")
and config["database_uuid"] in database_ids
@@ -92,7 +81,7 @@ class ImportChartsCommand(BaseCommand):
}
# import charts with the correct parent ref
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("charts/")
and config["dataset_uuid"] in dataset_info
@@ -100,66 +89,3 @@ class ImportChartsCommand(BaseCommand):
# update datasource id, type, and name
config.update(dataset_info[config["dataset_uuid"]])
import_chart(session, config, overwrite=True)
def run(self) -> None:
self.validate()
# rollback to prevent partial imports
try:
self._import_bundle(db.session)
db.session.commit()
except Exception:
db.session.rollback()
raise ChartImportError()
def validate(self) -> None:
exceptions: List[ValidationError] = []
# load existing databases so we can apply the password validation
db_passwords = {
str(uuid): password
for uuid, password in db.session.query(
Database.uuid, Database.password
).all()
}
# verify that the metadata file is present and valid
try:
metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
except ValidationError as exc:
exceptions.append(exc)
metadata = None
# validate charts, datasets, and databases
for file_name, content in self.contents.items():
prefix = file_name.split("/")[0]
schema = schemas.get(f"{prefix}/")
if schema:
try:
config = load_yaml(file_name, content)
# populate passwords from the request or from existing DBs
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
elif prefix == "databases" and config["uuid"] in db_passwords:
config["password"] = db_passwords[config["uuid"]]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:
exc.messages = {file_name: exc.messages}
exceptions.append(exc)
# validate that the type declared in METADATA_FILE_NAME is correct
if metadata:
type_validator = validate.Equal(Slice.__name__)
try:
type_validator(metadata["type"])
except ValidationError as exc:
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
exceptions.append(exc)
if exceptions:
exception = CommandInvalidError("Error importing chart")
exception.add_list(exceptions)
raise exception

6
superset/commands/export.py
View File

@@ -43,8 +43,8 @@ class ExportModelsCommand(BaseCommand):
self._models: List[Model] = []
@staticmethod
def export(model: Model) -> Iterator[Tuple[str, str]]:
raise NotImplementedError("Subclasses MUST implement export")
def _export(model: Model) -> Iterator[Tuple[str, str]]:
raise NotImplementedError("Subclasses MUST implement _export")
def run(self) -> Iterator[Tuple[str, str]]:
self.validate()
@@ -58,7 +58,7 @@ class ExportModelsCommand(BaseCommand):
seen = {METADATA_FILE_NAME}
for model in self._models:
for file_name, file_content in self.export(model):
for file_name, file_content in self._export(model):
if file_name not in seen:
yield file_name, file_content
seen.add(file_name)

116
superset/commands/importers/v1/__init__.py
View File

@@ -14,3 +14,119 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Any, Dict, List, Optional
from marshmallow import Schema, validate
from marshmallow.exceptions import ValidationError
from sqlalchemy.orm import Session
from superset import db
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandException, CommandInvalidError
from superset.commands.importers.v1.utils import (
load_metadata,
load_yaml,
METADATA_FILE_NAME,
)
from superset.dao.base import BaseDAO
from superset.models.core import Database
class ImportModelsCommand(BaseCommand):
"""Import models"""
dao = BaseDAO
model_name = "model"
schemas: Dict[str, Schema] = {}
import_error = CommandException
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
@staticmethod
def _import(session: Session, configs: Dict[str, Any]) -> None:
raise NotImplementedError("Subclasses MUSC implement _import")
def run(self) -> None:
self.validate()
# rollback to prevent partial imports
try:
self._import(db.session, self._configs)
db.session.commit()
except Exception:
db.session.rollback()
raise self.import_error()
def validate(self) -> None:
exceptions: List[ValidationError] = []
# load existing databases so we can apply the password validation
db_passwords = {
str(uuid): password
for uuid, password in db.session.query(
Database.uuid, Database.password
).all()
}
# verify that the metadata file is present and valid
try:
metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
except ValidationError as exc:
exceptions.append(exc)
metadata = None
# validate objects
for file_name, content in self.contents.items():
prefix = file_name.split("/")[0]
schema = self.schemas.get(f"{prefix}/")
if schema:
try:
config = load_yaml(file_name, content)
# populate passwords from the request or from existing DBs
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
elif prefix == "databases" and config["uuid"] in db_passwords:
config["password"] = db_passwords[config["uuid"]]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:
exc.messages = {file_name: exc.messages}
exceptions.append(exc)
# validate that the type declared in METADATA_FILE_NAME is correct
if metadata:
type_validator = validate.Equal(self.dao.model_cls.__name__) # type: ignore
try:
type_validator(metadata["type"])
except ValidationError as exc:
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
exceptions.append(exc)
if exceptions:
exception = CommandInvalidError(f"Error importing {self.model_name}")
exception.add_list(exceptions)
raise exception

2
superset/dashboards/commands/export.py
View File

@@ -43,7 +43,7 @@ class ExportDashboardsCommand(ExportModelsCommand):
not_found = DashboardNotFoundError
@staticmethod
def export(model: Dashboard) -> Iterator[Tuple[str, str]]:
def _export(model: Dashboard) -> Iterator[Tuple[str, str]]:
dashboard_slug = secure_filename(model.dashboard_title)
file_name = f"dashboards/{dashboard_slug}.yaml"

121
superset/dashboards/commands/importers/v1/__init__.py
View File

@@ -15,39 +15,24 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, Dict, Iterator, List, Optional, Set, Tuple
from typing import Any, Dict, Iterator, List, Set, Tuple
from marshmallow import Schema, validate
from marshmallow.exceptions import ValidationError
from marshmallow import Schema
from sqlalchemy.orm import Session
from sqlalchemy.sql import select
from superset import db
from superset.charts.commands.importers.v1.utils import import_chart
from superset.charts.schemas import ImportV1ChartSchema
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import (
load_metadata,
load_yaml,
METADATA_FILE_NAME,
)
from superset.commands.importers.v1 import ImportModelsCommand
from superset.dashboards.commands.exceptions import DashboardImportError
from superset.dashboards.commands.importers.v1.utils import import_dashboard
from superset.dashboards.dao import DashboardDAO
from superset.dashboards.schemas import ImportV1DashboardSchema
from superset.databases.commands.importers.v1.utils import import_database
from superset.databases.schemas import ImportV1DatabaseSchema
from superset.datasets.commands.importers.v1.utils import import_dataset
from superset.datasets.schemas import ImportV1DatasetSchema
from superset.models.core import Database
from superset.models.dashboard import Dashboard, dashboard_slices
schemas: Dict[str, Schema] = {
"charts/": ImportV1ChartSchema(),
"dashboards/": ImportV1DashboardSchema(),
"datasets/": ImportV1DatasetSchema(),
"databases/": ImportV1DatabaseSchema(),
}
from superset.models.dashboard import dashboard_slices
def find_chart_uuids(position: Dict[str, Any]) -> Iterator[str]:
@@ -61,47 +46,52 @@ def find_chart_uuids(position: Dict[str, Any]) -> Iterator[str]:
yield child["meta"]["uuid"]
class ImportDashboardsCommand(BaseCommand):
class ImportDashboardsCommand(ImportModelsCommand):
"""Import dashboards"""
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
dao = DashboardDAO
model_name = "dashboard"
schemas: Dict[str, Schema] = {
"charts/": ImportV1ChartSchema(),
"dashboards/": ImportV1DashboardSchema(),
"datasets/": ImportV1DatasetSchema(),
"databases/": ImportV1DatabaseSchema(),
}
import_error = DashboardImportError
# TODO (betodealmeida): refactor to use code from other commands
# pylint: disable=too-many-branches, too-many-locals
def _import_bundle(self, session: Session) -> None:
@staticmethod
def _import(session: Session, configs: Dict[str, Any]) -> None:
# discover charts associated with dashboards
chart_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("dashboards/"):
chart_uuids.update(find_chart_uuids(config["position"]))
# discover datasets associated with charts
dataset_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("charts/") and config["uuid"] in chart_uuids:
dataset_uuids.add(config["dataset_uuid"])
# discover databases associated with datasets
database_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("datasets/") and config["uuid"] in dataset_uuids:
database_uuids.add(config["database_uuid"])
# import related databases
database_ids: Dict[str, int] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("databases/") and config["uuid"] in database_uuids:
database = import_database(session, config, overwrite=False)
database_ids[str(database.uuid)] = database.id
# import datasets with the correct parent ref
dataset_info: Dict[str, Dict[str, Any]] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("datasets/")
and config["database_uuid"] in database_ids
@@ -116,7 +106,7 @@ class ImportDashboardsCommand(BaseCommand):
# import charts with the correct parent ref
chart_ids: Dict[str, int] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("charts/")
and config["dataset_uuid"] in dataset_info
@@ -133,7 +123,7 @@ class ImportDashboardsCommand(BaseCommand):
# import dashboards
dashboard_chart_ids: List[Tuple[int, int]] = []
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("dashboards/"):
dashboard = import_dashboard(session, config, overwrite=True)
@@ -149,66 +139,3 @@ class ImportDashboardsCommand(BaseCommand):
]
# pylint: disable=no-value-for-parameter (sqlalchemy/issues/4656)
session.execute(dashboard_slices.insert(), values)
def run(self) -> None:
self.validate()
# rollback to prevent partial imports
try:
self._import_bundle(db.session)
db.session.commit()
except Exception:
db.session.rollback()
raise DashboardImportError()
def validate(self) -> None:
exceptions: List[ValidationError] = []
# load existing databases so we can apply the password validation
db_passwords = {
str(uuid): password
for uuid, password in db.session.query(
Database.uuid, Database.password
).all()
}
# verify that the metadata file is present and valid
try:
metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
except ValidationError as exc:
exceptions.append(exc)
metadata = None
# validate dashboards, charts, datasets, and databases
for file_name, content in self.contents.items():
prefix = file_name.split("/")[0]
schema = schemas.get(f"{prefix}/")
if schema:
try:
config = load_yaml(file_name, content)
# populate passwords from the request or from existing DBs
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
elif prefix == "databases" and config["uuid"] in db_passwords:
config["password"] = db_passwords[config["uuid"]]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:
exc.messages = {file_name: exc.messages}
exceptions.append(exc)
# validate that the type declared in METADATA_FILE_NAME is correct
if metadata:
type_validator = validate.Equal(Dashboard.__name__)
try:
type_validator(metadata["type"])
except ValidationError as exc:
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
exceptions.append(exc)
if exceptions:
exception = CommandInvalidError("Error importing dashboard")
exception.add_list(exceptions)
raise exception

2
superset/databases/commands/export.py
View File

@@ -38,7 +38,7 @@ class ExportDatabasesCommand(ExportModelsCommand):
not_found = DatabaseNotFoundError
@staticmethod
def export(model: Database) -> Iterator[Tuple[str, str]]:
def _export(model: Database) -> Iterator[Tuple[str, str]]:
database_slug = secure_filename(model.database_name)
file_name = f"databases/{database_slug}.yaml"

105
superset/databases/commands/importers/v1/__init__.py
View File

@@ -15,53 +15,43 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, Dict, List, Optional
from typing import Any, Dict
from marshmallow import Schema, validate
from marshmallow.exceptions import ValidationError
from marshmallow import Schema
from sqlalchemy.orm import Session
from superset import db
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import (
load_metadata,
load_yaml,
METADATA_FILE_NAME,
)
from superset.commands.importers.v1 import ImportModelsCommand
from superset.databases.commands.exceptions import DatabaseImportError
from superset.databases.commands.importers.v1.utils import import_database
from superset.databases.dao import DatabaseDAO
from superset.databases.schemas import ImportV1DatabaseSchema
from superset.datasets.commands.importers.v1.utils import import_dataset
from superset.datasets.schemas import ImportV1DatasetSchema
from superset.models.core import Database
schemas: Dict[str, Schema] = {
"databases/": ImportV1DatabaseSchema(),
"datasets/": ImportV1DatasetSchema(),
}
class ImportDatabasesCommand(BaseCommand):
class ImportDatabasesCommand(ImportModelsCommand):
"""Import databases"""
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
dao = DatabaseDAO
model_name = "database"
schemas: Dict[str, Schema] = {
"databases/": ImportV1DatabaseSchema(),
"datasets/": ImportV1DatasetSchema(),
}
import_error = DatabaseImportError
def _import_bundle(self, session: Session) -> None:
@staticmethod
def _import(session: Session, configs: Dict[str, Any]) -> None:
# first import databases
database_ids: Dict[str, int] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("databases/"):
database = import_database(session, config, overwrite=True)
database_ids[str(database.uuid)] = database.id
# import related datasets
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("datasets/")
and config["database_uuid"] in database_ids
@@ -69,66 +59,3 @@ class ImportDatabasesCommand(BaseCommand):
config["database_id"] = database_ids[config["database_uuid"]]
# overwrite=False prevents deleting any non-imported columns/metrics
import_dataset(session, config, overwrite=False)
def run(self) -> None:
self.validate()
# rollback to prevent partial imports
try:
self._import_bundle(db.session)
db.session.commit()
except Exception:
db.session.rollback()
raise DatabaseImportError()
def validate(self) -> None:
exceptions: List[ValidationError] = []
# load existing databases so we can apply the password validation
db_passwords = {
str(uuid): password
for uuid, password in db.session.query(
Database.uuid, Database.password
).all()
}
# verify that the metadata file is present and valid
try:
metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
except ValidationError as exc:
exceptions.append(exc)
metadata = None
# validate databases and dataset
for file_name, content in self.contents.items():
prefix = file_name.split("/")[0]
schema = schemas.get(f"{prefix}/")
if schema:
try:
config = load_yaml(file_name, content)
# populate passwords from the request or from existing DBs
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
elif prefix == "databases" and config["uuid"] in db_passwords:
config["password"] = db_passwords[config["uuid"]]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:
exc.messages = {file_name: exc.messages}
exceptions.append(exc)
# validate that the type declared in METADATA_FILE_NAME is correct
if metadata:
type_validator = validate.Equal(Database.__name__)
try:
type_validator(metadata["type"])
except ValidationError as exc:
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
exceptions.append(exc)
if exceptions:
exception = CommandInvalidError("Error importing database")
exception.add_list(exceptions)
raise exception

2
superset/datasets/commands/export.py
View File

@@ -38,7 +38,7 @@ class ExportDatasetsCommand(ExportModelsCommand):
not_found = DatasetNotFoundError
@staticmethod
def export(model: SqlaTable) -> Iterator[Tuple[str, str]]:
def _export(model: SqlaTable) -> Iterator[Tuple[str, str]]:
database_slug = secure_filename(model.database.database_name)
dataset_slug = secure_filename(model.table_name)
file_name = f"datasets/{database_slug}/{dataset_slug}.yaml"

108
superset/datasets/commands/importers/v1/__init__.py
View File

@@ -15,126 +15,52 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, Dict, List, Optional, Set
from typing import Any, Dict, Set
from marshmallow import Schema, validate
from marshmallow.exceptions import ValidationError
from marshmallow import Schema
from sqlalchemy.orm import Session
from superset import db
from superset.commands.base import BaseCommand
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import (
load_metadata,
load_yaml,
METADATA_FILE_NAME,
)
from superset.connectors.sqla.models import SqlaTable
from superset.commands.importers.v1 import ImportModelsCommand
from superset.databases.commands.importers.v1.utils import import_database
from superset.databases.schemas import ImportV1DatabaseSchema
from superset.datasets.commands.exceptions import DatasetImportError
from superset.datasets.commands.importers.v1.utils import import_dataset
from superset.datasets.dao import DatasetDAO
from superset.datasets.schemas import ImportV1DatasetSchema
from superset.models.core import Database
schemas: Dict[str, Schema] = {
"databases/": ImportV1DatabaseSchema(),
"datasets/": ImportV1DatasetSchema(),
}
class ImportDatasetsCommand(BaseCommand):
class ImportDatasetsCommand(ImportModelsCommand):
"""Import datasets"""
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
dao = DatasetDAO
model_name = "dataset"
schemas: Dict[str, Schema] = {
"databases/": ImportV1DatabaseSchema(),
"datasets/": ImportV1DatasetSchema(),
}
import_error = DatasetImportError
def _import_bundle(self, session: Session) -> None:
@staticmethod
def _import(session: Session, configs: Dict[str, Any]) -> None:
# discover databases associated with datasets
database_uuids: Set[str] = set()
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("datasets/"):
database_uuids.add(config["database_uuid"])
# import related databases
database_ids: Dict[str, int] = {}
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if file_name.startswith("databases/") and config["uuid"] in database_uuids:
database = import_database(session, config, overwrite=False)
database_ids[str(database.uuid)] = database.id
# import datasets with the correct parent ref
for file_name, config in self._configs.items():
for file_name, config in configs.items():
if (
file_name.startswith("datasets/")
and config["database_uuid"] in database_ids
):
config["database_id"] = database_ids[config["database_uuid"]]
import_dataset(session, config, overwrite=True)
def run(self) -> None:
self.validate()
# rollback to prevent partial imports
try:
self._import_bundle(db.session)
db.session.commit()
except Exception:
db.session.rollback()
raise DatasetImportError()
def validate(self) -> None:
exceptions: List[ValidationError] = []
# load existing databases so we can apply the password validation
db_passwords = {
str(uuid): password
for uuid, password in db.session.query(
Database.uuid, Database.password
).all()
}
# verify that the metadata file is present and valid
try:
metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
except ValidationError as exc:
exceptions.append(exc)
metadata = None
# validate datasets and databases
for file_name, content in self.contents.items():
prefix = file_name.split("/")[0]
schema = schemas.get(f"{prefix}/")
if schema:
try:
config = load_yaml(file_name, content)
# populate passwords from the request or from existing DBs
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
elif prefix == "databases" and config["uuid"] in db_passwords:
config["password"] = db_passwords[config["uuid"]]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:
exc.messages = {file_name: exc.messages}
exceptions.append(exc)
# validate that the type declared in METADATA_FILE_NAME is correct
if metadata:
type_validator = validate.Equal(SqlaTable.__name__)
try:
type_validator(metadata["type"])
except ValidationError as exc:
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
exceptions.append(exc)
if exceptions:
exception = CommandInvalidError("Error importing dataset")
exception.add_list(exceptions)
raise exception

2
superset/queries/saved_queries/commands/export.py
View File

@@ -38,7 +38,7 @@ class ExportSavedQueriesCommand(ExportModelsCommand):
not_found = SavedQueryNotFoundError
@staticmethod
def export(model: SavedQuery) -> Iterator[Tuple[str, str]]:
def _export(model: SavedQuery) -> Iterator[Tuple[str, str]]:
# build filename based on database, optional schema, and label
database_slug = secure_filename(model.database.database_name)
schema_slug = secure_filename(model.schema)