feat: add modal to import databases (#11884)

* feat: add modal to import databases

* Fix test

* Improve hook

* Remove log and needless store.

* Change JS functions
This commit is contained in:
Beto Dealmeida
2020-12-07 11:22:45 -08:00
committed by GitHub
parent a7bba92469
commit 2b9695c520
13 changed files with 569 additions and 13 deletions

View File

@@ -0,0 +1,96 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import { styledMount as mount } from 'spec/helpers/theming';
import { ReactWrapper } from 'enzyme';
import ImportDatabaseModal from 'src/database/components/ImportModal';
import Modal from 'src/common/components/Modal';
const requiredProps = {
addDangerToast: () => {},
addSuccessToast: () => {},
onDatabaseImport: () => {},
show: true,
onHide: () => {},
};
describe('ImportDatabaseModal', () => {
let wrapper: ReactWrapper;
beforeEach(() => {
wrapper = mount(<ImportDatabaseModal {...requiredProps} />);
});
afterEach(() => {
jest.clearAllMocks();
});
it('renders', () => {
expect(wrapper.find(ImportDatabaseModal)).toExist();
});
it('renders a Modal', () => {
expect(wrapper.find(Modal)).toExist();
});
it('renders "Import Database" header', () => {
expect(wrapper.find('h4').text()).toEqual('Import Database');
});
it('renders a label and a file input field', () => {
expect(wrapper.find('input[type="file"]')).toExist();
expect(wrapper.find('label')).toExist();
});
it('should attach the label to the input field', () => {
const id = 'databaseFile';
expect(wrapper.find('label').prop('htmlFor')).toBe(id);
expect(wrapper.find('input').prop('id')).toBe(id);
});
it('should render the close, import and cancel buttons', () => {
expect(wrapper.find('button')).toHaveLength(3);
});
it('should render the import button initially disabled', () => {
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
true,
);
});
it('should render the import button enabled when a file is selected', () => {
const file = new File([new ArrayBuffer(1)], 'database_export.zip');
wrapper.find('input').simulate('change', { target: { files: [file] } });
expect(wrapper.find('button[children="Import"]').prop('disabled')).toBe(
false,
);
});
it('should render password fields when needed for import', () => {
const wrapperWithPasswords = mount(
<ImportDatabaseModal
{...requiredProps}
passwordFields={['databases/examples.yaml']}
/>,
);
expect(wrapperWithPasswords.find('input[type="password"]')).toExist();
});
});

View File

@@ -0,0 +1,191 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { t } from '@superset-ui/core';
import Modal from 'src/common/components/Modal';
import {
StyledIcon,
StyledInputContainer,
} from 'src/views/CRUD/data/database/DatabaseModal';
import { useImportResource } from 'src/views/CRUD/hooks';
import { DatabaseObject } from 'src/views/CRUD/data/database/types';
export interface ImportDatabaseModalProps {
addDangerToast: (msg: string) => void;
addSuccessToast: (msg: string) => void;
onDatabaseImport: () => void;
show: boolean;
onHide: () => void;
passwordFields?: string[];
setPasswordFields?: (passwordFields: string[]) => void;
}
const ImportDatabaseModal: FunctionComponent<ImportDatabaseModalProps> = ({
addDangerToast,
addSuccessToast,
onDatabaseImport,
show,
onHide,
passwordFields = [],
setPasswordFields = () => {},
}) => {
const [uploadFile, setUploadFile] = useState<File | null>(null);
const [isHidden, setIsHidden] = useState<boolean>(true);
const [passwords, setPasswords] = useState<Record<string, string>>({});
const fileInputRef = useRef<HTMLInputElement>(null);
const clearModal = () => {
setUploadFile(null);
setPasswords({});
setPasswordFields([]);
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.value = '';
}
};
const handleErrorMsg = (msg: string) => {
clearModal();
addDangerToast(msg);
};
const {
state: { passwordsNeeded },
importResource,
} = useImportResource<DatabaseObject>(
'database',
t('database'),
handleErrorMsg,
);
useEffect(() => {
setPasswordFields(passwordsNeeded);
}, [passwordsNeeded]);
// Functions
const hide = () => {
setIsHidden(true);
onHide();
};
const onUpload = () => {
if (uploadFile === null) {
return;
}
importResource(uploadFile, passwords).then(result => {
if (result) {
addSuccessToast(t('The databases have been imported'));
clearModal();
onDatabaseImport();
}
});
};
const changeFile = (event: React.ChangeEvent<HTMLInputElement>) => {
const { files } = event.target as HTMLInputElement;
setUploadFile((files && files[0]) || null);
};
const renderPasswordFields = () => {
if (passwordFields.length === 0) {
return null;
}
return (
<>
<h5>Passwords</h5>
<StyledInputContainer>
<div className="helper">
{t('Please provide the password for the databases below')}
</div>
</StyledInputContainer>
{passwordFields.map(fileName => (
<StyledInputContainer key={`password-for-${fileName}`}>
<div className="control-label">
{fileName}
<span className="required">*</span>
</div>
<input
name={`password-${fileName}`}
autoComplete="off"
type="password"
value={passwords[fileName]}
onChange={event =>
setPasswords({ ...passwords, [fileName]: event.target.value })
}
/>
</StyledInputContainer>
))}
</>
);
};
// Show/hide
if (isHidden && show) {
setIsHidden(false);
}
return (
<Modal
name="database"
className="database-modal"
disablePrimaryButton={uploadFile === null}
onHandledPrimaryAction={onUpload}
onHide={hide}
primaryButtonName={t('Import')}
width="750px"
show={show}
title={
<h4>
<StyledIcon name="database" />
{t('Import Database')}
</h4>
}
>
<StyledInputContainer>
<div className="control-label">
<label htmlFor="databaseFile">
{t('File')}
<span className="required">*</span>
</label>
</div>
<input
ref={fileInputRef}
data-test="database-file-input"
name="databaseFile"
id="databaseFile"
type="file"
accept=".yaml,.json,.yml,.zip"
onChange={changeFile}
/>
<div className="helper">
{t(
'Please note that the "Secure Extra" and "Certificate" sections of ' +
'the database configuration are not present in export files, and ' +
'should be added manually after the import if they are needed.',
)}
</div>
</StyledInputContainer>
{renderPasswordFields()}
</Modal>
);
};
export default ImportDatabaseModal;

View File

@@ -29,6 +29,7 @@ import TooltipWrapper from 'src/components/TooltipWrapper';
import Icon from 'src/components/Icon';
import ListView, { Filters } from 'src/components/ListView';
import { commonMenuData } from 'src/views/CRUD/data/common';
import ImportDatabaseModal from 'src/database/components/ImportModal/index';
import DatabaseModal from './DatabaseModal';
import { DatabaseObject } from './types';
@@ -74,6 +75,21 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) {
const [currentDatabase, setCurrentDatabase] = useState<DatabaseObject | null>(
null,
);
const [importingDatabase, showImportModal] = useState<boolean>(false);
const [passwordFields, setPasswordFields] = useState<string[]>([]);
const openDatabaseImportModal = () => {
showImportModal(true);
};
const closeDatabaseImportModal = () => {
showImportModal(false);
};
const handleDatabaseImport = () => {
showImportModal(false);
refreshData();
};
const openDatabaseDeleteModal = (database: DatabaseObject) =>
SupersetClient.get({
@@ -146,6 +162,14 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) {
},
},
];
if (isFeatureEnabled(FeatureFlag.VERSIONED_EXPORT)) {
menuData.buttons.push({
name: <Icon name="import" />,
buttonStyle: 'link',
onClick: openDatabaseImportModal,
});
}
}
function handleDatabaseExport(database: DatabaseObject) {
@@ -400,6 +424,16 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) {
loading={loading}
pageSize={PAGE_SIZE}
/>
<ImportDatabaseModal
show={importingDatabase}
onHide={closeDatabaseImportModal}
addDangerToast={addDangerToast}
addSuccessToast={addSuccessToast}
onDatabaseImport={handleDatabaseImport}
passwordFields={passwordFields}
setPasswordFields={setPasswordFields}
/>
</>
);
}

View File

@@ -39,11 +39,11 @@ interface DatabaseModalProps {
database?: DatabaseObject | null; // If included, will go into edit mode
}
const StyledIcon = styled(Icon)`
export const StyledIcon = styled(Icon)`
margin: auto ${({ theme }) => theme.gridUnit * 2}px auto 0;
`;
const StyledInputContainer = styled.div`
export const StyledInputContainer = styled.div`
margin-bottom: ${({ theme }) => theme.gridUnit * 2}px;
&.extra-container {

View File

@@ -25,6 +25,7 @@ import { FetchDataConfig } from 'src/components/ListView';
import { FilterValue } from 'src/components/ListView/types';
import Chart, { Slice } from 'src/types/Chart';
import copyTextToClipboard from 'src/utils/copy';
import getClientErrorObject from 'src/utils/getClientErrorObject';
import { FavoriteStatus } from './types';
interface ListViewResourceState<D extends object = any> {
@@ -312,6 +313,97 @@ export function useSingleViewResource<D extends object = any>(
};
}
interface ImportResourceState<D extends object = any> {
loading: boolean;
passwordsNeeded: string[];
}
export function useImportResource<D extends object = any>(
resourceName: string,
resourceLabel: string, // resourceLabel for translations
handleErrorMsg: (errorMsg: string) => void,
) {
const [state, setState] = useState<ImportResourceState<D>>({
loading: false,
passwordsNeeded: [],
});
function updateState(update: Partial<ImportResourceState<D>>) {
setState(currentState => ({ ...currentState, ...update }));
}
const needsPassword = (errMsg: Record<string, Record<string, string[]>>) =>
Object.values(errMsg).every(validationErrors =>
Object.entries(validationErrors as Object).every(
([field, messages]) =>
field === '_schema' &&
messages.length === 1 &&
messages[0] === 'Must provide a password for the database',
),
);
const importResource = useCallback(
(bundle: File, databasePasswords: Record<string, string> = {}) => {
// Set loading state
updateState({
loading: true,
});
const formData = new FormData();
formData.append('formData', bundle);
/* The import bundle never contains database passwords; if required
* they should be provided by the user during import.
*/
if (databasePasswords) {
formData.append('passwords', JSON.stringify(databasePasswords));
}
return SupersetClient.post({
endpoint: `/api/v1/${resourceName}/import/`,
body: formData,
})
.then(() => true)
.catch(response =>
getClientErrorObject(response).then(error => {
/* When importing a bundle, if all validation errors are because
* the databases need passwords we return a list of the database
* files so that the user can type in the passwords and resubmit
* the file.
*/
const errMsg = error.message || error.error;
if (typeof errMsg !== 'string' && needsPassword(errMsg)) {
updateState({
passwordsNeeded: Object.keys(errMsg),
});
return false;
}
handleErrorMsg(
t(
'An error occurred while importing %s: %s',
resourceLabel,
JSON.stringify(errMsg),
),
);
return false;
}),
)
.finally(() => {
updateState({ loading: false });
});
},
[],
);
return {
state: {
loading: state.loading,
passwordsNeeded: state.passwordsNeeded,
},
importResource,
};
}
enum FavStarClassName {
CHART = 'slice',
DASHBOARD = 'Dashboard',

View File

@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
import logging
from datetime import datetime
from io import BytesIO
@@ -776,7 +777,13 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
for file_name in bundle.namelist()
}
command = ImportDatabasesCommand(contents)
passwords = (
json.loads(request.form["passwords"])
if "passwords" in request.form
else None
)
command = ImportDatabasesCommand(contents, passwords=passwords)
try:
command.run()
return self.response(200, message="OK")

View File

@@ -41,12 +41,14 @@ class ImportDatabasesCommand(BaseCommand):
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
self.contents = contents
self.args = args
self.kwargs = kwargs
def run(self) -> None:
# iterate over all commands until we find a version that can
# handle the contents
for version in command_versions:
command = version(self.contents)
command = version(self.contents, *self.args, **self.kwargs)
try:
command.run()
return

View File

@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import urllib.parse
from typing import Any, Dict, List, Optional
from marshmallow import Schema, validate
@@ -47,8 +48,11 @@ class ImportDatabasesCommand(BaseCommand):
"""Import databases"""
# pylint: disable=unused-argument
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
def __init__(
self, contents: Dict[str, str], *args: Any, **kwargs: Any,
):
self.contents = contents
self.passwords = kwargs.get("passwords") or {}
self._configs: Dict[str, Any] = {}
def _import_bundle(self, session: Session) -> None:
@@ -96,6 +100,8 @@ class ImportDatabasesCommand(BaseCommand):
if schema:
try:
config = load_yaml(file_name, content)
if file_name in self.passwords:
config["password"] = self.passwords[file_name]
schema.load(config)
self._configs[file_name] = config
except ValidationError as exc:

View File

@@ -16,16 +16,19 @@
# under the License.
import inspect
import json
import urllib.parse
from typing import Any, Dict
from flask import current_app
from flask_babel import lazy_gettext as _
from marshmallow import fields, Schema
from marshmallow import fields, Schema, validates_schema
from marshmallow.validate import Length, ValidationError
from sqlalchemy import MetaData
from sqlalchemy.engine.url import make_url
from sqlalchemy.exc import ArgumentError
from superset.exceptions import CertificateException
from superset.models.core import PASSWORD_MASK
from superset.utils.core import markdown, parse_ssl_cert
database_schemas_query_schema = {
@@ -420,6 +423,7 @@ class ImportV1DatabaseExtraSchema(Schema):
class ImportV1DatabaseSchema(Schema):
database_name = fields.String(required=True)
sqlalchemy_uri = fields.String(required=True)
password = fields.String(allow_none=True)
cache_timeout = fields.Integer(allow_none=True)
expose_in_sqllab = fields.Boolean()
allow_run_async = fields.Boolean()
@@ -429,3 +433,12 @@ class ImportV1DatabaseSchema(Schema):
extra = fields.Nested(ImportV1DatabaseExtraSchema)
uuid = fields.UUID(required=True)
version = fields.String(required=True)
# pylint: disable=no-self-use, unused-argument
@validates_schema
def validate_password(self, data: Dict[str, Any], **kwargs: Any) -> None:
"""If sqlalchemy_uri has a masked password, password is required"""
uri = data["sqlalchemy_uri"]
password = urllib.parse.urlparse(uri).password
if password == PASSWORD_MASK and data.get("password") is None:
raise ValidationError("Must provide a password for the database")

View File

@@ -155,6 +155,7 @@ class Database(
"allow_csv_upload",
"extra",
]
extra_import_fields = ["password"]
export_children = ["tables"]
def __repr__(self) -> str:

View File

@@ -85,6 +85,10 @@ class ImportExportMixin:
# The names of the attributes
# that are available for import and export
extra_import_fields: List[str] = []
# Additional fields that should be imported,
# even though they were not exported
__mapper__: Mapper
@classmethod
@@ -155,7 +159,12 @@ class ImportExportMixin:
if sync is None:
sync = []
parent_refs = cls.parent_foreign_key_mappings()
export_fields = set(cls.export_fields) | set(parent_refs.keys()) | {"uuid"}
export_fields = (
set(cls.export_fields)
| set(cls.extra_import_fields)
| set(parent_refs.keys())
| {"uuid"}
)
new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
unique_constrains = cls._unique_constrains()

View File

@@ -158,7 +158,7 @@ class TestDatabaseApi(SupersetTestCase):
Database API: Test get items not allowed
"""
self.login(username="gamma")
uri = f"api/v1/database/"
uri = "api/v1/database/"
rv = self.client.get(uri)
self.assertEqual(rv.status_code, 200)
response = json.loads(rv.data.decode("utf-8"))
@@ -451,7 +451,7 @@ class TestDatabaseApi(SupersetTestCase):
"""
self.login(username="admin")
database_data = {"database_name": "test-database-updated"}
uri = f"api/v1/database/invalid"
uri = "api/v1/database/invalid"
rv = self.client.put(uri, json=database_data)
self.assertEqual(rv.status_code, 404)
@@ -556,7 +556,7 @@ class TestDatabaseApi(SupersetTestCase):
rv = self.client.get(uri)
self.assertEqual(rv.status_code, 404)
uri = f"api/v1/database/some_database/table/some_table/some_schema/"
uri = "api/v1/database/some_database/table/some_table/some_schema/"
rv = self.client.get(uri)
self.assertEqual(rv.status_code, 404)
@@ -718,7 +718,7 @@ class TestDatabaseApi(SupersetTestCase):
"sqlalchemy_uri": example_db.safe_sqlalchemy_uri(),
"server_cert": ssl_certificate,
}
url = f"api/v1/database/test_connection"
url = "api/v1/database/test_connection"
rv = self.post_assert_metric(url, data, "test_connection")
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv.headers["Content-Type"], "application/json; charset=utf-8")
@@ -747,7 +747,7 @@ class TestDatabaseApi(SupersetTestCase):
"impersonate_user": False,
"server_cert": None,
}
url = f"api/v1/database/test_connection"
url = "api/v1/database/test_connection"
rv = self.post_assert_metric(url, data, "test_connection")
self.assertEqual(rv.status_code, 400)
self.assertEqual(rv.headers["Content-Type"], "application/json; charset=utf-8")
@@ -787,7 +787,7 @@ class TestDatabaseApi(SupersetTestCase):
"impersonate_user": False,
"server_cert": None,
}
url = f"api/v1/database/test_connection"
url = "api/v1/database/test_connection"
rv = self.post_assert_metric(url, data, "test_connection")
self.assertEqual(rv.status_code, 400)
response = json.loads(rv.data.decode("utf-8"))
@@ -947,3 +947,88 @@ class TestDatabaseApi(SupersetTestCase):
assert response == {
"message": {"metadata.yaml": {"type": ["Must be equal to Database."]}}
}
def test_import_database_masked_password(self):
"""
Database API: Test import database with masked password
"""
self.login(username="admin")
uri = "api/v1/database/import/"
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"
] = "postgresql://username:XXXXXXXXXX@host:12345/db"
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open(
"database_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(masked_database_config).encode())
with bundle.open(
"database_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"formData": (buf, "database_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 422
assert response == {
"message": {
"databases/imported_database.yaml": {
"_schema": ["Must provide a password for the database"]
}
}
}
def test_import_database_masked_password_provided(self):
"""
Database API: Test import database with masked password provided
"""
self.login(username="admin")
uri = "api/v1/database/import/"
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"
] = "postgresql://username:XXXXXXXXXX@host:12345/db"
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open(
"database_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(masked_database_config).encode())
buf.seek(0)
form_data = {
"formData": (buf, "database_export.zip"),
"passwords": json.dumps({"databases/imported_database.yaml": "SECRET"}),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 200
assert response == {"message": "OK"}
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert database.database_name == "imported_database"
assert (
database.sqlalchemy_uri == "postgresql://username:XXXXXXXXXX@host:12345/db"
)
assert database.password == "SECRET"
db.session.delete(database)
db.session.commit()

View File

@@ -452,6 +452,26 @@ class TestImportDatabasesCommand(SupersetTestCase):
}
}
def test_import_v1_database_masked_password(self):
"""Test that database imports with masked passwords are rejected"""
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"
] = "postgresql://username:XXXXXXXXXX@host:12345/db"
contents = {
"metadata.yaml": yaml.safe_dump(database_metadata_config),
"databases/imported_database.yaml": yaml.safe_dump(masked_database_config),
}
command = ImportDatabasesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing database"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"_schema": ["Must provide a password for the database"]
}
}
@patch("superset.databases.commands.importers.v1.import_dataset")
def test_import_v1_rollback(self, mock_import_dataset):
"""Test than on an exception everything is rolled back"""