diff --git a/superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/components/ImportModal/ImportModal.test.tsx similarity index 81% rename from superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx rename to superset-frontend/src/components/ImportModal/ImportModal.test.tsx index ddcf44e820e..1e922b4d128 100644 --- a/superset-frontend/src/chart/components/ImportModal/ImportModal.test.tsx +++ b/superset-frontend/src/components/ImportModal/ImportModal.test.tsx @@ -22,25 +22,29 @@ import configureStore from 'redux-mock-store'; import { styledMount as mount } from 'spec/helpers/theming'; import { ReactWrapper } from 'enzyme'; -import ImportChartModal from 'src/chart/components/ImportModal'; +import ImportModelsModal, { StyledIcon } from 'src/components/ImportModal'; import Modal from 'src/common/components/Modal'; const mockStore = configureStore([thunk]); const store = mockStore({}); const requiredProps = { + resourceName: 'model', + resourceLabel: 'model', + icon: , + passwordsNeededMessage: 'Passwords are needed', addDangerToast: () => {}, addSuccessToast: () => {}, - onChartImport: () => {}, + onModelImport: () => {}, show: true, onHide: () => {}, }; -describe('ImportChartModal', () => { +describe('ImportModelsModal', () => { let wrapper: ReactWrapper; beforeEach(() => { - wrapper = mount(, { + wrapper = mount(, { context: { store }, }); }); @@ -50,15 +54,15 @@ describe('ImportChartModal', () => { }); it('renders', () => { - expect(wrapper.find(ImportChartModal)).toExist(); + expect(wrapper.find(ImportModelsModal)).toExist(); }); it('renders a Modal', () => { expect(wrapper.find(Modal)).toExist(); }); - it('renders "Import Chart" header', () => { - expect(wrapper.find('h4').text()).toEqual('Import Chart'); + it('renders "Import model" header', () => { + expect(wrapper.find('h4').text()).toEqual('Import model'); }); it('renders a label and a file input field', () => { @@ -67,7 +71,7 @@ describe('ImportChartModal', () => { }); it('should attach the label to the input field', () => { - const id = 'chartFile'; + const id = 'modelFile'; expect(wrapper.find('label').prop('htmlFor')).toBe(id); expect(wrapper.find('input').prop('id')).toBe(id); }); @@ -83,7 +87,7 @@ describe('ImportChartModal', () => { }); it('should render the import button enabled when a file is selected', () => { - const file = new File([new ArrayBuffer(1)], 'chart_export.zip'); + const file = new File([new ArrayBuffer(1)], 'model_export.zip'); wrapper.find('input').simulate('change', { target: { files: [file] } }); expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( @@ -93,7 +97,7 @@ describe('ImportChartModal', () => { it('should render password fields when needed for import', () => { const wrapperWithPasswords = mount( - , diff --git a/superset-frontend/src/chart/components/ImportModal/index.tsx b/superset-frontend/src/components/ImportModal/index.tsx similarity index 61% rename from superset-frontend/src/chart/components/ImportModal/index.tsx rename to superset-frontend/src/components/ImportModal/index.tsx index 328af1885f6..1c26623f10c 100644 --- a/superset-frontend/src/chart/components/ImportModal/index.tsx +++ b/superset-frontend/src/components/ImportModal/index.tsx @@ -17,30 +17,107 @@ * under the License. */ import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; -import { t } from '@superset-ui/core'; +import { styled, t } from '@superset-ui/core'; +import Icon from 'src//components/Icon'; import Modal from 'src/common/components/Modal'; -import { - StyledIcon, - StyledInputContainer, -} from 'src/views/CRUD/data/database/DatabaseModal'; import { useImportResource } from 'src/views/CRUD/hooks'; -import { ChartObject } from 'src/views/CRUD/chart/types'; -export interface ImportChartModalProps { +export const StyledIcon = styled(Icon)` + margin: auto ${({ theme }) => theme.gridUnit * 2}px auto 0; +`; + +const StyledInputContainer = styled.div` + margin-bottom: ${({ theme }) => theme.gridUnit * 2}px; + + &.extra-container { + padding-top: 8px; + } + + .helper { + display: block; + padding: ${({ theme }) => theme.gridUnit}px 0; + color: ${({ theme }) => theme.colors.grayscale.base}; + font-size: ${({ theme }) => theme.typography.sizes.s - 1}px; + text-align: left; + + .required { + margin-left: ${({ theme }) => theme.gridUnit / 2}px; + color: ${({ theme }) => theme.colors.error.base}; + } + } + + .input-container { + display: flex; + align-items: center; + + label { + display: flex; + margin-right: ${({ theme }) => theme.gridUnit * 2}px; + } + + i { + margin: 0 ${({ theme }) => theme.gridUnit}px; + } + } + + input, + textarea { + flex: 1 1 auto; + } + + textarea { + height: 160px; + resize: none; + } + + input::placeholder, + textarea::placeholder { + color: ${({ theme }) => theme.colors.grayscale.light1}; + } + + textarea, + input[type='text'], + input[type='number'] { + padding: ${({ theme }) => theme.gridUnit * 1.5}px + ${({ theme }) => theme.gridUnit * 2}px; + border-style: none; + border: 1px solid ${({ theme }) => theme.colors.grayscale.light2}; + border-radius: ${({ theme }) => theme.gridUnit}px; + + &[name='name'] { + flex: 0 1 auto; + width: 40%; + } + + &[name='sqlalchemy_uri'] { + margin-right: ${({ theme }) => theme.gridUnit * 3}px; + } + } +`; + +export interface ImportModelsModalProps { + resourceName: string; + resourceLabel: string; + icon: React.ReactNode; + passwordsNeededMessage: string; addDangerToast: (msg: string) => void; addSuccessToast: (msg: string) => void; - onChartImport: () => void; + onModelImport: () => void; show: boolean; onHide: () => void; passwordFields?: string[]; setPasswordFields?: (passwordFields: string[]) => void; } -const ImportChartModal: FunctionComponent = ({ +const ImportModelsModal: FunctionComponent = ({ + resourceName, + resourceLabel, + icon, + passwordsNeededMessage, addDangerToast, addSuccessToast, - onChartImport, + onModelImport, show, onHide, passwordFields = [], @@ -68,11 +145,11 @@ const ImportChartModal: FunctionComponent = ({ const { state: { passwordsNeeded }, importResource, - } = useImportResource('chart', t('chart'), handleErrorMsg); + } = useImportResource(resourceName, resourceLabel, handleErrorMsg); useEffect(() => { setPasswordFields(passwordsNeeded); - }, [passwordsNeeded]); + }, [passwordsNeeded, setPasswordFields]); // Functions const hide = () => { @@ -87,9 +164,9 @@ const ImportChartModal: FunctionComponent = ({ importResource(uploadFile, passwords).then(result => { if (result) { - addSuccessToast(t('The charts have been imported')); + addSuccessToast(t('The import was successful')); clearModal(); - onChartImport(); + onModelImport(); } }); }; @@ -108,15 +185,7 @@ const ImportChartModal: FunctionComponent = ({ <>
Database passwords
-
- {t( - 'The passwords for the databases below are needed in order to ' + - 'import them together with the charts. Please note that the ' + - '"Secure Extra" and "Certificate" sections of ' + - 'the database configuration are not present in export files, and ' + - 'should be added manually after the import if they are needed.', - )} -
+
{passwordsNeededMessage}
 {passwordFields.map(fileName => ( @@ -126,7 +195,7 @@ const ImportChartModal: FunctionComponent = ({ @@ -146,8 +215,8 @@ const ImportChartModal: FunctionComponent = ({ return ( = ({ show={show} title={

- - {t('Import Chart')} + {icon} + {t('Import %s', resourceLabel)}

} >
-
= ({ ); }; -export default ImportChartModal; +export default ImportModelsModal; diff --git a/superset-frontend/src/dashboard/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/dashboard/components/ImportModal/ImportModal.test.tsx deleted file mode 100644 index ac46fd65af2..00000000000 --- a/superset-frontend/src/dashboard/components/ImportModal/ImportModal.test.tsx +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React from 'react'; -import thunk from 'redux-thunk'; -import configureStore from 'redux-mock-store'; -import { styledMount as mount } from 'spec/helpers/theming'; -import { ReactWrapper } from 'enzyme'; - -import ImportDashboardModal from 'src/dashboard/components/ImportModal'; -import Modal from 'src/common/components/Modal'; - -const mockStore = configureStore([thunk]); -const store = mockStore({}); - -const requiredProps = { - addDangerToast: () => {}, - addSuccessToast: () => {}, - onDashboardImport: () => {}, - show: true, - onHide: () => {}, -}; - -describe('ImportDashboardModal', () => { - let wrapper: ReactWrapper; - - beforeEach(() => { - wrapper = mount(, { - context: { store }, - }); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('renders', () => { - expect(wrapper.find(ImportDashboardModal)).toExist(); - }); - - it('renders a Modal', () => { - expect(wrapper.find(Modal)).toExist(); - }); - - it('renders "Import Dashboard" header', () => { - expect(wrapper.find('h4').text()).toEqual('Import Dashboard'); - }); - - it('renders a label and a file input field', () => { - expect(wrapper.find('input[type="file"]')).toExist(); - expect(wrapper.find('label')).toExist(); - }); - - it('should attach the label to the input field', () => { - const id = 'dashboardFile'; - expect(wrapper.find('label').prop('htmlFor')).toBe(id); - expect(wrapper.find('input').prop('id')).toBe(id); - }); - - it('should render the close, import and cancel buttons', () => { - expect(wrapper.find('button')).toHaveLength(3); - }); - - it('should render the import button initially disabled', () => { - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - true, - ); - }); - - it('should render the import button enabled when a file is selected', () => { - const file = new File([new ArrayBuffer(1)], 'dashboard_export.zip'); - wrapper.find('input').simulate('change', { target: { files: [file] } }); - - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - false, - ); - }); - - it('should render password fields when needed for import', () => { - const wrapperWithPasswords = mount( - , - { - context: { store }, - }, - ); - expect(wrapperWithPasswords.find('input[type="password"]')).toExist(); - }); -}); diff --git a/superset-frontend/src/dashboard/components/ImportModal/index.tsx b/superset-frontend/src/dashboard/components/ImportModal/index.tsx deleted file mode 100644 index 0d8cf262e80..00000000000 --- a/superset-frontend/src/dashboard/components/ImportModal/index.tsx +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; -import { t } from '@superset-ui/core'; - -import Modal from 'src/common/components/Modal'; -import { - StyledIcon, - StyledInputContainer, -} from 'src/views/CRUD/data/database/DatabaseModal'; -import { useImportResource } from 'src/views/CRUD/hooks'; -import { DashboardObject } from 'src/views/CRUD/dashboard/types'; - -export interface ImportDashboardModalProps { - addDangerToast: (msg: string) => void; - addSuccessToast: (msg: string) => void; - onDashboardImport: () => void; - show: boolean; - onHide: () => void; - passwordFields?: string[]; - setPasswordFields?: (passwordFields: string[]) => void; -} - -const ImportDashboardModal: FunctionComponent = ({ - addDangerToast, - addSuccessToast, - onDashboardImport, - show, - onHide, - passwordFields = [], - setPasswordFields = () => {}, -}) => { - const [uploadFile, setUploadFile] = useState(null); - const [isHidden, setIsHidden] = useState(true); - const [passwords, setPasswords] = useState>({}); - const fileInputRef = useRef(null); - - const clearModal = () => { - setUploadFile(null); - setPasswordFields([]); - setPasswords({}); - if (fileInputRef && fileInputRef.current) { - fileInputRef.current.value = ''; - } - }; - - const handleErrorMsg = (msg: string) => { - clearModal(); - addDangerToast(msg); - }; - - const { - state: { passwordsNeeded }, - importResource, - } = useImportResource( - 'dashboard', - t('dashboard'), - handleErrorMsg, - ); - - useEffect(() => { - setPasswordFields(passwordsNeeded); - }, [passwordsNeeded]); - - // Functions - const hide = () => { - setIsHidden(true); - onHide(); - }; - - const onUpload = () => { - if (uploadFile === null) { - return; - } - - importResource(uploadFile, passwords).then(result => { - if (result) { - addSuccessToast(t('The dashboards have been imported')); - clearModal(); - onDashboardImport(); - } - }); - }; - - const changeFile = (event: React.ChangeEvent) => { - const { files } = event.target as HTMLInputElement; - setUploadFile((files && files[0]) || null); - }; - - const renderPasswordFields = () => { - if (passwordFields.length === 0) { - return null; - } - - return ( - <> -
Database passwords
- -
- {t( - 'The passwords for the databases below are needed in order to ' + - 'import them together with the dashboards. Please note that the ' + - '"Secure Extra" and "Certificate" sections of ' + - 'the database configuration are not present in export files, and ' + - 'should be added manually after the import if they are needed.', - )} -
- - {passwordFields.map(fileName => ( - -
- {fileName} - * -
- - setPasswords({ ...passwords, [fileName]: event.target.value }) - } - /> - - ))} - - ); - }; - - // Show/hide - if (isHidden && show) { - setIsHidden(false); - } - - return ( - - - {t('Import Dashboard')} - - } - > - -
- -
- - - {renderPasswordFields()} - - ); -}; - -export default ImportDashboardModal; diff --git a/superset-frontend/src/database/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/database/components/ImportModal/ImportModal.test.tsx deleted file mode 100644 index acad00c3923..00000000000 --- a/superset-frontend/src/database/components/ImportModal/ImportModal.test.tsx +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React from 'react'; -import { styledMount as mount } from 'spec/helpers/theming'; -import { ReactWrapper } from 'enzyme'; - -import ImportDatabaseModal from 'src/database/components/ImportModal'; -import Modal from 'src/common/components/Modal'; - -const requiredProps = { - addDangerToast: () => {}, - addSuccessToast: () => {}, - onDatabaseImport: () => {}, - show: true, - onHide: () => {}, -}; - -describe('ImportDatabaseModal', () => { - let wrapper: ReactWrapper; - - beforeEach(() => { - wrapper = mount(); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('renders', () => { - expect(wrapper.find(ImportDatabaseModal)).toExist(); - }); - - it('renders a Modal', () => { - expect(wrapper.find(Modal)).toExist(); - }); - - it('renders "Import Database" header', () => { - expect(wrapper.find('h4').text()).toEqual('Import Database'); - }); - - it('renders a label and a file input field', () => { - expect(wrapper.find('input[type="file"]')).toExist(); - expect(wrapper.find('label')).toExist(); - }); - - it('should attach the label to the input field', () => { - const id = 'databaseFile'; - expect(wrapper.find('label').prop('htmlFor')).toBe(id); - expect(wrapper.find('input').prop('id')).toBe(id); - }); - - it('should render the close, import and cancel buttons', () => { - expect(wrapper.find('button')).toHaveLength(3); - }); - - it('should render the import button initially disabled', () => { - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - true, - ); - }); - - it('should render the import button enabled when a file is selected', () => { - const file = new File([new ArrayBuffer(1)], 'database_export.zip'); - wrapper.find('input').simulate('change', { target: { files: [file] } }); - - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - false, - ); - }); - - it('should render password fields when needed for import', () => { - const wrapperWithPasswords = mount( - , - ); - expect(wrapperWithPasswords.find('input[type="password"]')).toExist(); - }); -}); diff --git a/superset-frontend/src/database/components/ImportModal/index.tsx b/superset-frontend/src/database/components/ImportModal/index.tsx deleted file mode 100644 index 35234d28649..00000000000 --- a/superset-frontend/src/database/components/ImportModal/index.tsx +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; -import { t } from '@superset-ui/core'; - -import Modal from 'src/common/components/Modal'; -import { - StyledIcon, - StyledInputContainer, -} from 'src/views/CRUD/data/database/DatabaseModal'; -import { useImportResource } from 'src/views/CRUD/hooks'; -import { DatabaseObject } from 'src/views/CRUD/data/database/types'; - -export interface ImportDatabaseModalProps { - addDangerToast: (msg: string) => void; - addSuccessToast: (msg: string) => void; - onDatabaseImport: () => void; - show: boolean; - onHide: () => void; - passwordFields?: string[]; - setPasswordFields?: (passwordFields: string[]) => void; -} - -const ImportDatabaseModal: FunctionComponent = ({ - addDangerToast, - addSuccessToast, - onDatabaseImport, - show, - onHide, - passwordFields = [], - setPasswordFields = () => {}, -}) => { - const [uploadFile, setUploadFile] = useState(null); - const [isHidden, setIsHidden] = useState(true); - const [passwords, setPasswords] = useState>({}); - const fileInputRef = useRef(null); - - const clearModal = () => { - setUploadFile(null); - setPasswords({}); - setPasswordFields([]); - if (fileInputRef && fileInputRef.current) { - fileInputRef.current.value = ''; - } - }; - - const handleErrorMsg = (msg: string) => { - clearModal(); - addDangerToast(msg); - }; - - const { - state: { passwordsNeeded }, - importResource, - } = useImportResource( - 'database', - t('database'), - handleErrorMsg, - ); - - useEffect(() => { - setPasswordFields(passwordsNeeded); - }, [passwordsNeeded]); - - // Functions - const hide = () => { - setIsHidden(true); - onHide(); - }; - - const onUpload = () => { - if (uploadFile === null) { - return; - } - - importResource(uploadFile, passwords).then(result => { - if (result) { - addSuccessToast(t('The databases have been imported')); - clearModal(); - onDatabaseImport(); - } - }); - }; - - const changeFile = (event: React.ChangeEvent) => { - const { files } = event.target as HTMLInputElement; - setUploadFile((files && files[0]) || null); - }; - - const renderPasswordFields = () => { - if (passwordFields.length === 0) { - return null; - } - - return ( - <> -
Passwords
- -
- {t('Please provide the password for the databases below')} -
- - {passwordFields.map(fileName => ( - -
- {fileName} - * -
- - setPasswords({ ...passwords, [fileName]: event.target.value }) - } - /> - - ))} - - ); - }; - - // Show/hide - if (isHidden && show) { - setIsHidden(false); - } - - return ( - - - {t('Import Database')} - - } - > - -
- -
- -
- {t( - 'Please note that the "Secure Extra" and "Certificate" sections of ' + - 'the database configuration are not present in export files, and ' + - 'should be added manually after the import if they are needed.', - )} -
- - {renderPasswordFields()} - - ); -}; - -export default ImportDatabaseModal; diff --git a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx b/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx deleted file mode 100644 index 70871299e83..00000000000 --- a/superset-frontend/src/datasource/components/ImportModal/ImportModal.test.tsx +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React from 'react'; -import thunk from 'redux-thunk'; -import configureStore from 'redux-mock-store'; -import { styledMount as mount } from 'spec/helpers/theming'; -import { ReactWrapper } from 'enzyme'; - -import ImportDatasetModal from 'src/datasource/components/ImportModal'; -import Modal from 'src/common/components/Modal'; - -const mockStore = configureStore([thunk]); -const store = mockStore({}); - -const requiredProps = { - addDangerToast: () => {}, - addSuccessToast: () => {}, - onDatasetImport: () => {}, - show: true, - onHide: () => {}, -}; - -describe('ImportDatasetModal', () => { - let wrapper: ReactWrapper; - - beforeEach(() => { - wrapper = mount(, { - context: { store }, - }); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('renders', () => { - expect(wrapper.find(ImportDatasetModal)).toExist(); - }); - - it('renders a Modal', () => { - expect(wrapper.find(Modal)).toExist(); - }); - - it('renders "Import Dataset" header', () => { - expect(wrapper.find('h4').text()).toEqual('Import Dataset'); - }); - - it('renders a label and a file input field', () => { - expect(wrapper.find('input[type="file"]')).toExist(); - expect(wrapper.find('label')).toExist(); - }); - - it('should attach the label to the input field', () => { - const id = 'datasetFile'; - expect(wrapper.find('label').prop('htmlFor')).toBe(id); - expect(wrapper.find('input').prop('id')).toBe(id); - }); - - it('should render the close, import and cancel buttons', () => { - expect(wrapper.find('button')).toHaveLength(3); - }); - - it('should render the import button initially disabled', () => { - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - true, - ); - }); - - it('should render the import button enabled when a file is selected', () => { - const file = new File([new ArrayBuffer(1)], 'dataset_export.zip'); - wrapper.find('input').simulate('change', { target: { files: [file] } }); - - expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe( - false, - ); - }); - - it('should render password fields when needed for import', () => { - const wrapperWithPasswords = mount( - , - { - context: { store }, - }, - ); - expect(wrapperWithPasswords.find('input[type="password"]')).toExist(); - }); -}); diff --git a/superset-frontend/src/datasource/components/ImportModal/index.tsx b/superset-frontend/src/datasource/components/ImportModal/index.tsx deleted file mode 100644 index d8599e86ffc..00000000000 --- a/superset-frontend/src/datasource/components/ImportModal/index.tsx +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; -import { t } from '@superset-ui/core'; - -import Modal from 'src/common/components/Modal'; -import { - StyledIcon, - StyledInputContainer, -} from 'src/views/CRUD/data/database/DatabaseModal'; -import { useImportResource } from 'src/views/CRUD/hooks'; -import { DatasetObject } from 'src/views/CRUD/data/dataset/types'; - -export interface ImportDatasetModalProps { - addDangerToast: (msg: string) => void; - addSuccessToast: (msg: string) => void; - onDatasetImport: () => void; - show: boolean; - onHide: () => void; - passwordFields?: string[]; - setPasswordFields?: (passwordFields: string[]) => void; -} - -const ImportDatasetModal: FunctionComponent = ({ - addDangerToast, - addSuccessToast, - onDatasetImport, - show, - onHide, - passwordFields = [], - setPasswordFields = () => {}, -}) => { - const [uploadFile, setUploadFile] = useState(null); - const [isHidden, setIsHidden] = useState(true); - const [passwords, setPasswords] = useState>({}); - const fileInputRef = useRef(null); - - const clearModal = () => { - setUploadFile(null); - setPasswordFields([]); - setPasswords({}); - if (fileInputRef && fileInputRef.current) { - fileInputRef.current.value = ''; - } - }; - - const handleErrorMsg = (msg: string) => { - clearModal(); - addDangerToast(msg); - }; - - const { - state: { passwordsNeeded }, - importResource, - } = useImportResource('dataset', t('dataset'), handleErrorMsg); - - useEffect(() => { - setPasswordFields(passwordsNeeded); - }, [passwordsNeeded]); - - // Functions - const hide = () => { - setIsHidden(true); - onHide(); - }; - - const onUpload = () => { - if (uploadFile === null) { - return; - } - - importResource(uploadFile, passwords).then(result => { - if (result) { - addSuccessToast(t('The datasets have been imported')); - clearModal(); - onDatasetImport(); - } - }); - }; - - const changeFile = (event: React.ChangeEvent) => { - const { files } = event.target as HTMLInputElement; - setUploadFile((files && files[0]) || null); - }; - - const renderPasswordFields = () => { - if (passwordFields.length === 0) { - return null; - } - - return ( - <> -
Database passwords
- -
- {t( - 'The passwords for the databases below are needed in order to ' + - 'import them together with the datasets. Please note that the ' + - '"Secure Extra" and "Certificate" sections of ' + - 'the database configuration are not present in export files, and ' + - 'should be added manually after the import if they are needed.', - )} -
- - {passwordFields.map(fileName => ( - -
- {fileName} - * -
- - setPasswords({ ...passwords, [fileName]: event.target.value }) - } - /> - - ))} - - ); - }; - - // Show/hide - if (isHidden && show) { - setIsHidden(false); - } - - return ( - - - {t('Import Dataset')} - - } - > - -
- -
- - - {renderPasswordFields()} - - ); -}; - -export default ImportDatasetModal; diff --git a/superset-frontend/src/views/CRUD/chart/ChartList.tsx b/superset-frontend/src/views/CRUD/chart/ChartList.tsx index cf907be3fb5..4687ed6193b 100644 --- a/superset-frontend/src/views/CRUD/chart/ChartList.tsx +++ b/superset-frontend/src/views/CRUD/chart/ChartList.tsx @@ -43,7 +43,9 @@ import ListView, { } from 'src/components/ListView'; import withToasts from 'src/messageToasts/enhancers/withToasts'; import PropertiesModal from 'src/explore/components/PropertiesModal'; -import ImportChartModal from 'src/chart/components/ImportModal/index'; +import ImportModelsModal, { + StyledIcon, +} from 'src/components/ImportModal/index'; import Chart from 'src/types/Chart'; import TooltipWrapper from 'src/components/TooltipWrapper'; import ChartCard from './ChartCard'; @@ -130,13 +132,13 @@ function ChartList(props: ChartListProps) { const [importingChart, showImportModal] = useState(false); const [passwordFields, setPasswordFields] = useState([]); - function openChartImportModal() { + const openChartImportModal = () => { showImportModal(true); - } + }; - function closeChartImportModal() { + const closeChartImportModal = () => { showImportModal(false); - } + }; const handleChartImport = () => { showImportModal(false); @@ -568,12 +570,22 @@ function ChartList(props: ChartListProps) { }} - } + passwordsNeededMessage={t( + 'The passwords for the databases below are needed in order to ' + + 'import them together with the charts. Please note that the ' + + '"Secure Extra" and "Certificate" sections of ' + + 'the database configuration are not present in export files, and ' + + 'should be added manually after the import if they are needed.', + )} addDangerToast={addDangerToast} addSuccessToast={addSuccessToast} - onChartImport={handleChartImport} + onModelImport={handleChartImport} + show={importingChart} + onHide={closeChartImportModal} passwordFields={passwordFields} setPasswordFields={setPasswordFields} /> diff --git a/superset-frontend/src/views/CRUD/dashboard/DashboardList.tsx b/superset-frontend/src/views/CRUD/dashboard/DashboardList.tsx index e6f73885e97..e3fe98a0e93 100644 --- a/superset-frontend/src/views/CRUD/dashboard/DashboardList.tsx +++ b/superset-frontend/src/views/CRUD/dashboard/DashboardList.tsx @@ -37,7 +37,9 @@ import Icon from 'src/components/Icon'; import FaveStar from 'src/components/FaveStar'; import PropertiesModal from 'src/dashboard/components/PropertiesModal'; import TooltipWrapper from 'src/components/TooltipWrapper'; -import ImportDashboardModal from 'src/dashboard/components/ImportModal/index'; +import ImportModelsModal, { + StyledIcon, +} from 'src/components/ImportModal/index'; import Dashboard from 'src/dashboard/containers/Dashboard'; import DashboardCard from './DashboardCard'; @@ -67,6 +69,8 @@ interface Dashboard { } function DashboardList(props: DashboardListProps) { + const { addDangerToast, addSuccessToast } = props; + const { state: { loading, @@ -82,13 +86,13 @@ function DashboardList(props: DashboardListProps) { } = useListViewResource( 'dashboard', t('dashboard'), - props.addDangerToast, + addDangerToast, ); const dashboardIds = useMemo(() => dashboards.map(d => d.id), [dashboards]); const [saveFavoriteStatus, favoriteStatus] = useFavoriteStatus( 'dashboard', dashboardIds, - props.addDangerToast, + addDangerToast, ); const [dashboardToEdit, setDashboardToEdit] = useState( null, @@ -97,13 +101,13 @@ function DashboardList(props: DashboardListProps) { const [importingDashboard, showImportModal] = useState(false); const [passwordFields, setPasswordFields] = useState([]); - function openDashboardImportModal() { + const openDashboardImportModal = () => { showImportModal(true); - } + }; - function closeDashboardImportModal() { + const closeDashboardImportModal = () => { showImportModal(false); - } + }; const handleDashboardImport = () => { showImportModal(false); @@ -136,7 +140,7 @@ function DashboardList(props: DashboardListProps) { ); }, createErrorHandler(errMsg => - props.addDangerToast( + addDangerToast( t('An error occurred while fetching dashboards: %s', errMsg), ), ), @@ -150,10 +154,10 @@ function DashboardList(props: DashboardListProps) { )}`, }).then( ({ json = {} }) => { - props.addSuccessToast(json.message); + addSuccessToast(json.message); }, createErrorHandler(errMsg => - props.addDangerToast( + addDangerToast( t('There was an issue deleting the selected dashboards: ', errMsg), ), ), @@ -251,8 +255,8 @@ function DashboardList(props: DashboardListProps) { handleDashboardDelete( original, refreshData, - props.addSuccessToast, - props.addDangerToast, + addSuccessToast, + addDangerToast, ); const handleEdit = () => openDashboardEditModal(original); const handleExport = () => handleBulkDashboardExport([original]); @@ -346,7 +350,7 @@ function DashboardList(props: DashboardListProps) { 'dashboard', 'owners', createErrorHandler(errMsg => - props.addDangerToast( + addDangerToast( t( 'An error occurred while fetching dashboard owner values: %s', errMsg, @@ -367,7 +371,7 @@ function DashboardList(props: DashboardListProps) { 'dashboard', 'created_by', createErrorHandler(errMsg => - props.addDangerToast( + addDangerToast( t( 'An error occurred while fetching dashboard created by values: %s', errMsg, @@ -426,8 +430,8 @@ function DashboardList(props: DashboardListProps) { bulkSelectEnabled={bulkSelectEnabled} refreshData={refreshData} loading={loading} - addDangerToast={props.addDangerToast} - addSuccessToast={props.addSuccessToast} + addDangerToast={addDangerToast} + addSuccessToast={addSuccessToast} openDashboardEditModal={openDashboardEditModal} saveFavoriteStatus={saveFavoriteStatus} favoriteStatus={favoriteStatus[dashboard.id]} @@ -526,16 +530,26 @@ function DashboardList(props: DashboardListProps) { ); }} - } + passwordsNeededMessage={t( + 'The passwords for the databases below are needed in order to ' + + 'import them together with the dashboards. Please note that the ' + + '"Secure Extra" and "Certificate" sections of ' + + 'the database configuration are not present in export files, and ' + + 'should be added manually after the import if they are needed.', + )} + addDangerToast={addDangerToast} + addSuccessToast={addSuccessToast} + onModelImport={handleDashboardImport} show={importingDashboard} onHide={closeDashboardImportModal} - addDangerToast={props.addDangerToast} - addSuccessToast={props.addSuccessToast} - onDashboardImport={handleDashboardImport} passwordFields={passwordFields} setPasswordFields={setPasswordFields} /> - : ); } diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx index 4b0c3f836f2..f00fd8c99f5 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx @@ -29,7 +29,9 @@ import TooltipWrapper from 'src/components/TooltipWrapper'; import Icon from 'src/components/Icon'; import ListView, { Filters } from 'src/components/ListView'; import { commonMenuData } from 'src/views/CRUD/data/common'; -import ImportDatabaseModal from 'src/database/components/ImportModal/index'; +import ImportModelsModal, { + StyledIcon, +} from 'src/components/ImportModal/index'; import DatabaseModal from './DatabaseModal'; import { DatabaseObject } from './types'; @@ -425,12 +427,21 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) { pageSize={PAGE_SIZE} /> - } + passwordsNeededMessage={t( + 'The passwords for the databases below are needed in order to ' + + 'import them. Please note that the "Secure Extra" and "Certificate" ' + + 'sections of the database configuration are not present in export ' + + 'files, and should be added manually after the import if they are needed.', + )} addDangerToast={addDangerToast} addSuccessToast={addSuccessToast} - onDatabaseImport={handleDatabaseImport} + onModelImport={handleDatabaseImport} + show={importingDatabase} + onHide={closeDatabaseImportModal} passwordFields={passwordFields} setPasswordFields={setPasswordFields} /> diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal.tsx index 69fa99b599d..fa8a3625ab9 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal.tsx @@ -39,11 +39,11 @@ interface DatabaseModalProps { database?: DatabaseObject | null; // If included, will go into edit mode } -export const StyledIcon = styled(Icon)` +const StyledIcon = styled(Icon)` margin: auto ${({ theme }) => theme.gridUnit * 2}px auto 0; `; -export const StyledInputContainer = styled.div` +const StyledInputContainer = styled.div` margin-bottom: ${({ theme }) => theme.gridUnit * 2}px; &.extra-container { diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 34982c39723..82bbefb8811 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -45,7 +45,9 @@ import TooltipWrapper from 'src/components/TooltipWrapper'; import Icon from 'src/components/Icon'; import FacePile from 'src/components/FacePile'; import CertifiedIconWithTooltip from 'src/components/CertifiedIconWithTooltip'; -import ImportDatasetModal from 'src/datasource/components/ImportModal/index'; +import ImportModelsModal, { + StyledIcon, +} from 'src/components/ImportModal/index'; import { isFeatureEnabled, FeatureFlag } from 'src/featureFlags'; import AddDatasetModal from './AddDatasetModal'; @@ -647,12 +649,22 @@ const DatasetList: FunctionComponent = ({ }} - } + passwordsNeededMessage={t( + 'The passwords for the databases below are needed in order to ' + + 'import them together with the datasets. Please note that the ' + + '"Secure Extra" and "Certificate" sections of ' + + 'the database configuration are not present in export files, and ' + + 'should be added manually after the import if they are needed.', + )} addDangerToast={addDangerToast} addSuccessToast={addSuccessToast} - onDatasetImport={handleDatasetImport} + onModelImport={handleDatasetImport} + show={importingDataset} + onHide={closeDatasetImportModal} passwordFields={passwordFields} setPasswordFields={setPasswordFields} /> diff --git a/superset/app.py b/superset/app.py index 14e68052e5c..bdba4cc7f51 100644 --- a/superset/app.py +++ b/superset/app.py @@ -315,15 +315,16 @@ class SupersetAppInitializer: # # Add links # - appbuilder.add_link( - "Import Dashboards", - label=__("Import Dashboards"), - href="/superset/import_dashboards", - icon="fa-cloud-upload", - category="Manage", - category_label=__("Manage"), - category_icon="fa-wrench", - ) + if not feature_flag_manager.is_feature_enabled("VERSIONED_EXPORT"): + appbuilder.add_link( + "Import Dashboards", + label=__("Import Dashboards"), + href="/superset/import_dashboards", + icon="fa-cloud-upload", + category="Manage", + category_label=__("Manage"), + category_icon="fa-wrench", + ) appbuilder.add_link( "SQL Editor", label=_("SQL Editor"), diff --git a/superset/charts/commands/export.py b/superset/charts/commands/export.py index 12ca90722ab..405a679fde0 100644 --- a/superset/charts/commands/export.py +++ b/superset/charts/commands/export.py @@ -43,7 +43,7 @@ class ExportChartsCommand(ExportModelsCommand): not_found = ChartNotFoundError @staticmethod - def export(model: Slice) -> Iterator[Tuple[str, str]]: + def _export(model: Slice) -> Iterator[Tuple[str, str]]: chart_slug = secure_filename(model.slice_name) file_name = f"charts/{chart_slug}.yaml" diff --git a/superset/charts/commands/importers/v1/__init__.py b/superset/charts/commands/importers/v1/__init__.py index 69840f6c8f3..62dde9ff0e3 100644 --- a/superset/charts/commands/importers/v1/__init__.py +++ b/superset/charts/commands/importers/v1/__init__.py @@ -15,70 +15,59 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, Set -from marshmallow import Schema, validate -from marshmallow.exceptions import ValidationError +from marshmallow import Schema from sqlalchemy.orm import Session -from superset import db from superset.charts.commands.exceptions import ChartImportError from superset.charts.commands.importers.v1.utils import import_chart +from superset.charts.dao import ChartDAO from superset.charts.schemas import ImportV1ChartSchema -from superset.commands.base import BaseCommand -from superset.commands.exceptions import CommandInvalidError -from superset.commands.importers.v1.utils import ( - load_metadata, - load_yaml, - METADATA_FILE_NAME, -) +from superset.commands.importers.v1 import ImportModelsCommand from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema -from superset.models.core import Database -from superset.models.slice import Slice - -schemas: Dict[str, Schema] = { - "charts/": ImportV1ChartSchema(), - "datasets/": ImportV1DatasetSchema(), - "databases/": ImportV1DatabaseSchema(), -} -class ImportChartsCommand(BaseCommand): +class ImportChartsCommand(ImportModelsCommand): """Import charts""" - # pylint: disable=unused-argument - def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): - self.contents = contents - self.passwords: Dict[str, str] = kwargs.get("passwords") or {} - self._configs: Dict[str, Any] = {} + dao = ChartDAO + model_name = "chart" + schemas: Dict[str, Schema] = { + "charts/": ImportV1ChartSchema(), + "datasets/": ImportV1DatasetSchema(), + "databases/": ImportV1DatabaseSchema(), + } + import_error = ChartImportError - def _import_bundle(self, session: Session) -> None: + @staticmethod + def _import(session: Session, configs: Dict[str, Any]) -> None: # discover datasets associated with charts dataset_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("charts/"): dataset_uuids.add(config["dataset_uuid"]) # discover databases associated with datasets database_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("datasets/") and config["uuid"] in dataset_uuids: database_uuids.add(config["database_uuid"]) # import related databases database_ids: Dict[str, int] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("databases/") and config["uuid"] in database_uuids: database = import_database(session, config, overwrite=False) database_ids[str(database.uuid)] = database.id # import datasets with the correct parent ref dataset_info: Dict[str, Dict[str, Any]] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("datasets/") and config["database_uuid"] in database_ids @@ -92,7 +81,7 @@ class ImportChartsCommand(BaseCommand): } # import charts with the correct parent ref - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("charts/") and config["dataset_uuid"] in dataset_info @@ -100,66 +89,3 @@ class ImportChartsCommand(BaseCommand): # update datasource id, type, and name config.update(dataset_info[config["dataset_uuid"]]) import_chart(session, config, overwrite=True) - - def run(self) -> None: - self.validate() - - # rollback to prevent partial imports - try: - self._import_bundle(db.session) - db.session.commit() - except Exception: - db.session.rollback() - raise ChartImportError() - - def validate(self) -> None: - exceptions: List[ValidationError] = [] - - # load existing databases so we can apply the password validation - db_passwords = { - str(uuid): password - for uuid, password in db.session.query( - Database.uuid, Database.password - ).all() - } - - # verify that the metadata file is present and valid - try: - metadata: Optional[Dict[str, str]] = load_metadata(self.contents) - except ValidationError as exc: - exceptions.append(exc) - metadata = None - - # validate charts, datasets, and databases - for file_name, content in self.contents.items(): - prefix = file_name.split("/")[0] - schema = schemas.get(f"{prefix}/") - if schema: - try: - config = load_yaml(file_name, content) - - # populate passwords from the request or from existing DBs - if file_name in self.passwords: - config["password"] = self.passwords[file_name] - elif prefix == "databases" and config["uuid"] in db_passwords: - config["password"] = db_passwords[config["uuid"]] - - schema.load(config) - self._configs[file_name] = config - except ValidationError as exc: - exc.messages = {file_name: exc.messages} - exceptions.append(exc) - - # validate that the type declared in METADATA_FILE_NAME is correct - if metadata: - type_validator = validate.Equal(Slice.__name__) - try: - type_validator(metadata["type"]) - except ValidationError as exc: - exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}} - exceptions.append(exc) - - if exceptions: - exception = CommandInvalidError("Error importing chart") - exception.add_list(exceptions) - raise exception diff --git a/superset/commands/export.py b/superset/commands/export.py index 6c40af1dff2..5bf117cca31 100644 --- a/superset/commands/export.py +++ b/superset/commands/export.py @@ -43,8 +43,8 @@ class ExportModelsCommand(BaseCommand): self._models: List[Model] = [] @staticmethod - def export(model: Model) -> Iterator[Tuple[str, str]]: - raise NotImplementedError("Subclasses MUST implement export") + def _export(model: Model) -> Iterator[Tuple[str, str]]: + raise NotImplementedError("Subclasses MUST implement _export") def run(self) -> Iterator[Tuple[str, str]]: self.validate() @@ -58,7 +58,7 @@ class ExportModelsCommand(BaseCommand): seen = {METADATA_FILE_NAME} for model in self._models: - for file_name, file_content in self.export(model): + for file_name, file_content in self._export(model): if file_name not in seen: yield file_name, file_content seen.add(file_name) diff --git a/superset/commands/importers/v1/__init__.py b/superset/commands/importers/v1/__init__.py index 13a83393a91..16d3314e116 100644 --- a/superset/commands/importers/v1/__init__.py +++ b/superset/commands/importers/v1/__init__.py @@ -14,3 +14,119 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Dict, List, Optional + +from marshmallow import Schema, validate +from marshmallow.exceptions import ValidationError +from sqlalchemy.orm import Session + +from superset import db +from superset.commands.base import BaseCommand +from superset.commands.exceptions import CommandException, CommandInvalidError +from superset.commands.importers.v1.utils import ( + load_metadata, + load_yaml, + METADATA_FILE_NAME, +) +from superset.dao.base import BaseDAO +from superset.models.core import Database + + +class ImportModelsCommand(BaseCommand): + + """Import models""" + + dao = BaseDAO + model_name = "model" + schemas: Dict[str, Schema] = {} + import_error = CommandException + + # pylint: disable=unused-argument + def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): + self.contents = contents + self.passwords: Dict[str, str] = kwargs.get("passwords") or {} + self._configs: Dict[str, Any] = {} + + @staticmethod + def _import(session: Session, configs: Dict[str, Any]) -> None: + raise NotImplementedError("Subclasses MUSC implement _import") + + def run(self) -> None: + self.validate() + + # rollback to prevent partial imports + try: + self._import(db.session, self._configs) + db.session.commit() + except Exception: + db.session.rollback() + raise self.import_error() + + def validate(self) -> None: + exceptions: List[ValidationError] = [] + + # load existing databases so we can apply the password validation + db_passwords = { + str(uuid): password + for uuid, password in db.session.query( + Database.uuid, Database.password + ).all() + } + + # verify that the metadata file is present and valid + try: + metadata: Optional[Dict[str, str]] = load_metadata(self.contents) + except ValidationError as exc: + exceptions.append(exc) + metadata = None + + # validate objects + for file_name, content in self.contents.items(): + prefix = file_name.split("/")[0] + schema = self.schemas.get(f"{prefix}/") + if schema: + try: + config = load_yaml(file_name, content) + + # populate passwords from the request or from existing DBs + if file_name in self.passwords: + config["password"] = self.passwords[file_name] + elif prefix == "databases" and config["uuid"] in db_passwords: + config["password"] = db_passwords[config["uuid"]] + + schema.load(config) + self._configs[file_name] = config + except ValidationError as exc: + exc.messages = {file_name: exc.messages} + exceptions.append(exc) + + # validate that the type declared in METADATA_FILE_NAME is correct + if metadata: + type_validator = validate.Equal(self.dao.model_cls.__name__) # type: ignore + try: + type_validator(metadata["type"]) + except ValidationError as exc: + exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}} + exceptions.append(exc) + + if exceptions: + exception = CommandInvalidError(f"Error importing {self.model_name}") + exception.add_list(exceptions) + raise exception diff --git a/superset/dashboards/commands/export.py b/superset/dashboards/commands/export.py index 345c0a74a90..b3867758e9f 100644 --- a/superset/dashboards/commands/export.py +++ b/superset/dashboards/commands/export.py @@ -43,7 +43,7 @@ class ExportDashboardsCommand(ExportModelsCommand): not_found = DashboardNotFoundError @staticmethod - def export(model: Dashboard) -> Iterator[Tuple[str, str]]: + def _export(model: Dashboard) -> Iterator[Tuple[str, str]]: dashboard_slug = secure_filename(model.dashboard_title) file_name = f"dashboards/{dashboard_slug}.yaml" diff --git a/superset/dashboards/commands/importers/v1/__init__.py b/superset/dashboards/commands/importers/v1/__init__.py index 67f1e7790af..0b7b235d310 100644 --- a/superset/dashboards/commands/importers/v1/__init__.py +++ b/superset/dashboards/commands/importers/v1/__init__.py @@ -15,39 +15,24 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, Iterator, List, Optional, Set, Tuple +from typing import Any, Dict, Iterator, List, Set, Tuple -from marshmallow import Schema, validate -from marshmallow.exceptions import ValidationError +from marshmallow import Schema from sqlalchemy.orm import Session from sqlalchemy.sql import select -from superset import db from superset.charts.commands.importers.v1.utils import import_chart from superset.charts.schemas import ImportV1ChartSchema -from superset.commands.base import BaseCommand -from superset.commands.exceptions import CommandInvalidError -from superset.commands.importers.v1.utils import ( - load_metadata, - load_yaml, - METADATA_FILE_NAME, -) +from superset.commands.importers.v1 import ImportModelsCommand from superset.dashboards.commands.exceptions import DashboardImportError from superset.dashboards.commands.importers.v1.utils import import_dashboard +from superset.dashboards.dao import DashboardDAO from superset.dashboards.schemas import ImportV1DashboardSchema from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema -from superset.models.core import Database -from superset.models.dashboard import Dashboard, dashboard_slices - -schemas: Dict[str, Schema] = { - "charts/": ImportV1ChartSchema(), - "dashboards/": ImportV1DashboardSchema(), - "datasets/": ImportV1DatasetSchema(), - "databases/": ImportV1DatabaseSchema(), -} +from superset.models.dashboard import dashboard_slices def find_chart_uuids(position: Dict[str, Any]) -> Iterator[str]: @@ -61,47 +46,52 @@ def find_chart_uuids(position: Dict[str, Any]) -> Iterator[str]: yield child["meta"]["uuid"] -class ImportDashboardsCommand(BaseCommand): +class ImportDashboardsCommand(ImportModelsCommand): """Import dashboards""" - # pylint: disable=unused-argument - def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): - self.contents = contents - self.passwords: Dict[str, str] = kwargs.get("passwords") or {} - self._configs: Dict[str, Any] = {} + dao = DashboardDAO + model_name = "dashboard" + schemas: Dict[str, Schema] = { + "charts/": ImportV1ChartSchema(), + "dashboards/": ImportV1DashboardSchema(), + "datasets/": ImportV1DatasetSchema(), + "databases/": ImportV1DatabaseSchema(), + } + import_error = DashboardImportError # TODO (betodealmeida): refactor to use code from other commands # pylint: disable=too-many-branches, too-many-locals - def _import_bundle(self, session: Session) -> None: + @staticmethod + def _import(session: Session, configs: Dict[str, Any]) -> None: # discover charts associated with dashboards chart_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("dashboards/"): chart_uuids.update(find_chart_uuids(config["position"])) # discover datasets associated with charts dataset_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("charts/") and config["uuid"] in chart_uuids: dataset_uuids.add(config["dataset_uuid"]) # discover databases associated with datasets database_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("datasets/") and config["uuid"] in dataset_uuids: database_uuids.add(config["database_uuid"]) # import related databases database_ids: Dict[str, int] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("databases/") and config["uuid"] in database_uuids: database = import_database(session, config, overwrite=False) database_ids[str(database.uuid)] = database.id # import datasets with the correct parent ref dataset_info: Dict[str, Dict[str, Any]] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("datasets/") and config["database_uuid"] in database_ids @@ -116,7 +106,7 @@ class ImportDashboardsCommand(BaseCommand): # import charts with the correct parent ref chart_ids: Dict[str, int] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("charts/") and config["dataset_uuid"] in dataset_info @@ -133,7 +123,7 @@ class ImportDashboardsCommand(BaseCommand): # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("dashboards/"): dashboard = import_dashboard(session, config, overwrite=True) @@ -149,66 +139,3 @@ class ImportDashboardsCommand(BaseCommand): ] # pylint: disable=no-value-for-parameter (sqlalchemy/issues/4656) session.execute(dashboard_slices.insert(), values) - - def run(self) -> None: - self.validate() - - # rollback to prevent partial imports - try: - self._import_bundle(db.session) - db.session.commit() - except Exception: - db.session.rollback() - raise DashboardImportError() - - def validate(self) -> None: - exceptions: List[ValidationError] = [] - - # load existing databases so we can apply the password validation - db_passwords = { - str(uuid): password - for uuid, password in db.session.query( - Database.uuid, Database.password - ).all() - } - - # verify that the metadata file is present and valid - try: - metadata: Optional[Dict[str, str]] = load_metadata(self.contents) - except ValidationError as exc: - exceptions.append(exc) - metadata = None - - # validate dashboards, charts, datasets, and databases - for file_name, content in self.contents.items(): - prefix = file_name.split("/")[0] - schema = schemas.get(f"{prefix}/") - if schema: - try: - config = load_yaml(file_name, content) - - # populate passwords from the request or from existing DBs - if file_name in self.passwords: - config["password"] = self.passwords[file_name] - elif prefix == "databases" and config["uuid"] in db_passwords: - config["password"] = db_passwords[config["uuid"]] - - schema.load(config) - self._configs[file_name] = config - except ValidationError as exc: - exc.messages = {file_name: exc.messages} - exceptions.append(exc) - - # validate that the type declared in METADATA_FILE_NAME is correct - if metadata: - type_validator = validate.Equal(Dashboard.__name__) - try: - type_validator(metadata["type"]) - except ValidationError as exc: - exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}} - exceptions.append(exc) - - if exceptions: - exception = CommandInvalidError("Error importing dashboard") - exception.add_list(exceptions) - raise exception diff --git a/superset/databases/commands/export.py b/superset/databases/commands/export.py index 9fc2e5e51d0..e8937867d1e 100644 --- a/superset/databases/commands/export.py +++ b/superset/databases/commands/export.py @@ -38,7 +38,7 @@ class ExportDatabasesCommand(ExportModelsCommand): not_found = DatabaseNotFoundError @staticmethod - def export(model: Database) -> Iterator[Tuple[str, str]]: + def _export(model: Database) -> Iterator[Tuple[str, str]]: database_slug = secure_filename(model.database_name) file_name = f"databases/{database_slug}.yaml" diff --git a/superset/databases/commands/importers/v1/__init__.py b/superset/databases/commands/importers/v1/__init__.py index cf9c4bc1d9b..6453b877deb 100644 --- a/superset/databases/commands/importers/v1/__init__.py +++ b/superset/databases/commands/importers/v1/__init__.py @@ -15,53 +15,43 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, List, Optional +from typing import Any, Dict -from marshmallow import Schema, validate -from marshmallow.exceptions import ValidationError +from marshmallow import Schema from sqlalchemy.orm import Session -from superset import db -from superset.commands.base import BaseCommand -from superset.commands.exceptions import CommandInvalidError -from superset.commands.importers.v1.utils import ( - load_metadata, - load_yaml, - METADATA_FILE_NAME, -) +from superset.commands.importers.v1 import ImportModelsCommand from superset.databases.commands.exceptions import DatabaseImportError from superset.databases.commands.importers.v1.utils import import_database +from superset.databases.dao import DatabaseDAO from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema -from superset.models.core import Database - -schemas: Dict[str, Schema] = { - "databases/": ImportV1DatabaseSchema(), - "datasets/": ImportV1DatasetSchema(), -} -class ImportDatabasesCommand(BaseCommand): +class ImportDatabasesCommand(ImportModelsCommand): """Import databases""" - # pylint: disable=unused-argument - def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): - self.contents = contents - self.passwords: Dict[str, str] = kwargs.get("passwords") or {} - self._configs: Dict[str, Any] = {} + dao = DatabaseDAO + model_name = "database" + schemas: Dict[str, Schema] = { + "databases/": ImportV1DatabaseSchema(), + "datasets/": ImportV1DatasetSchema(), + } + import_error = DatabaseImportError - def _import_bundle(self, session: Session) -> None: + @staticmethod + def _import(session: Session, configs: Dict[str, Any]) -> None: # first import databases database_ids: Dict[str, int] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("databases/"): database = import_database(session, config, overwrite=True) database_ids[str(database.uuid)] = database.id # import related datasets - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("datasets/") and config["database_uuid"] in database_ids @@ -69,66 +59,3 @@ class ImportDatabasesCommand(BaseCommand): config["database_id"] = database_ids[config["database_uuid"]] # overwrite=False prevents deleting any non-imported columns/metrics import_dataset(session, config, overwrite=False) - - def run(self) -> None: - self.validate() - - # rollback to prevent partial imports - try: - self._import_bundle(db.session) - db.session.commit() - except Exception: - db.session.rollback() - raise DatabaseImportError() - - def validate(self) -> None: - exceptions: List[ValidationError] = [] - - # load existing databases so we can apply the password validation - db_passwords = { - str(uuid): password - for uuid, password in db.session.query( - Database.uuid, Database.password - ).all() - } - - # verify that the metadata file is present and valid - try: - metadata: Optional[Dict[str, str]] = load_metadata(self.contents) - except ValidationError as exc: - exceptions.append(exc) - metadata = None - - # validate databases and dataset - for file_name, content in self.contents.items(): - prefix = file_name.split("/")[0] - schema = schemas.get(f"{prefix}/") - if schema: - try: - config = load_yaml(file_name, content) - - # populate passwords from the request or from existing DBs - if file_name in self.passwords: - config["password"] = self.passwords[file_name] - elif prefix == "databases" and config["uuid"] in db_passwords: - config["password"] = db_passwords[config["uuid"]] - - schema.load(config) - self._configs[file_name] = config - except ValidationError as exc: - exc.messages = {file_name: exc.messages} - exceptions.append(exc) - - # validate that the type declared in METADATA_FILE_NAME is correct - if metadata: - type_validator = validate.Equal(Database.__name__) - try: - type_validator(metadata["type"]) - except ValidationError as exc: - exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}} - exceptions.append(exc) - - if exceptions: - exception = CommandInvalidError("Error importing database") - exception.add_list(exceptions) - raise exception diff --git a/superset/datasets/commands/export.py b/superset/datasets/commands/export.py index 72e417098d5..e86b9322956 100644 --- a/superset/datasets/commands/export.py +++ b/superset/datasets/commands/export.py @@ -38,7 +38,7 @@ class ExportDatasetsCommand(ExportModelsCommand): not_found = DatasetNotFoundError @staticmethod - def export(model: SqlaTable) -> Iterator[Tuple[str, str]]: + def _export(model: SqlaTable) -> Iterator[Tuple[str, str]]: database_slug = secure_filename(model.database.database_name) dataset_slug = secure_filename(model.table_name) file_name = f"datasets/{database_slug}/{dataset_slug}.yaml" diff --git a/superset/datasets/commands/importers/v1/__init__.py b/superset/datasets/commands/importers/v1/__init__.py index 200fdc316e8..81f363165fc 100644 --- a/superset/datasets/commands/importers/v1/__init__.py +++ b/superset/datasets/commands/importers/v1/__init__.py @@ -15,126 +15,52 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, Set -from marshmallow import Schema, validate -from marshmallow.exceptions import ValidationError +from marshmallow import Schema from sqlalchemy.orm import Session -from superset import db -from superset.commands.base import BaseCommand -from superset.commands.exceptions import CommandInvalidError -from superset.commands.importers.v1.utils import ( - load_metadata, - load_yaml, - METADATA_FILE_NAME, -) -from superset.connectors.sqla.models import SqlaTable +from superset.commands.importers.v1 import ImportModelsCommand from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.exceptions import DatasetImportError from superset.datasets.commands.importers.v1.utils import import_dataset +from superset.datasets.dao import DatasetDAO from superset.datasets.schemas import ImportV1DatasetSchema -from superset.models.core import Database - -schemas: Dict[str, Schema] = { - "databases/": ImportV1DatabaseSchema(), - "datasets/": ImportV1DatasetSchema(), -} -class ImportDatasetsCommand(BaseCommand): +class ImportDatasetsCommand(ImportModelsCommand): """Import datasets""" - # pylint: disable=unused-argument - def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): - self.contents = contents - self.passwords: Dict[str, str] = kwargs.get("passwords") or {} - self._configs: Dict[str, Any] = {} + dao = DatasetDAO + model_name = "dataset" + schemas: Dict[str, Schema] = { + "databases/": ImportV1DatabaseSchema(), + "datasets/": ImportV1DatasetSchema(), + } + import_error = DatasetImportError - def _import_bundle(self, session: Session) -> None: + @staticmethod + def _import(session: Session, configs: Dict[str, Any]) -> None: # discover databases associated with datasets database_uuids: Set[str] = set() - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("datasets/"): database_uuids.add(config["database_uuid"]) # import related databases database_ids: Dict[str, int] = {} - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if file_name.startswith("databases/") and config["uuid"] in database_uuids: database = import_database(session, config, overwrite=False) database_ids[str(database.uuid)] = database.id # import datasets with the correct parent ref - for file_name, config in self._configs.items(): + for file_name, config in configs.items(): if ( file_name.startswith("datasets/") and config["database_uuid"] in database_ids ): config["database_id"] = database_ids[config["database_uuid"]] import_dataset(session, config, overwrite=True) - - def run(self) -> None: - self.validate() - - # rollback to prevent partial imports - try: - self._import_bundle(db.session) - db.session.commit() - except Exception: - db.session.rollback() - raise DatasetImportError() - - def validate(self) -> None: - exceptions: List[ValidationError] = [] - - # load existing databases so we can apply the password validation - db_passwords = { - str(uuid): password - for uuid, password in db.session.query( - Database.uuid, Database.password - ).all() - } - - # verify that the metadata file is present and valid - try: - metadata: Optional[Dict[str, str]] = load_metadata(self.contents) - except ValidationError as exc: - exceptions.append(exc) - metadata = None - - # validate datasets and databases - for file_name, content in self.contents.items(): - prefix = file_name.split("/")[0] - schema = schemas.get(f"{prefix}/") - if schema: - try: - config = load_yaml(file_name, content) - - # populate passwords from the request or from existing DBs - if file_name in self.passwords: - config["password"] = self.passwords[file_name] - elif prefix == "databases" and config["uuid"] in db_passwords: - config["password"] = db_passwords[config["uuid"]] - - schema.load(config) - self._configs[file_name] = config - except ValidationError as exc: - exc.messages = {file_name: exc.messages} - exceptions.append(exc) - - # validate that the type declared in METADATA_FILE_NAME is correct - if metadata: - type_validator = validate.Equal(SqlaTable.__name__) - try: - type_validator(metadata["type"]) - except ValidationError as exc: - exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}} - exceptions.append(exc) - - if exceptions: - exception = CommandInvalidError("Error importing dataset") - exception.add_list(exceptions) - raise exception diff --git a/superset/queries/saved_queries/commands/export.py b/superset/queries/saved_queries/commands/export.py index c22155ea871..ca2cfe5de96 100644 --- a/superset/queries/saved_queries/commands/export.py +++ b/superset/queries/saved_queries/commands/export.py @@ -38,7 +38,7 @@ class ExportSavedQueriesCommand(ExportModelsCommand): not_found = SavedQueryNotFoundError @staticmethod - def export(model: SavedQuery) -> Iterator[Tuple[str, str]]: + def _export(model: SavedQuery) -> Iterator[Tuple[str, str]]: # build filename based on database, optional schema, and label database_slug = secure_filename(model.database.database_name) schema_slug = secure_filename(model.schema)