QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: API for asset sync (#19220)

Browse Source 
* feat: API for asset sync

* Add unit tests.

* Improve tests

* Move files

* Add more tests
This commit is contained in:
Beto Dealmeida
2022-03-22 08:29:24 -07:00
committed by GitHub
parent d3ce398448
commit b05e7dbf2a
7 changed files with 490 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tests/unit_tests/conftest.py
View File

@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-outer-name, import-outside-toplevel
from typing import Iterator
import importlib
from typing import Any, Iterator
import pytest
from pytest_mock import MockFixture
@@ -25,11 +26,12 @@ from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.session import Session
from superset.app import SupersetApp
from superset.extensions import appbuilder
from superset.initialization import SupersetAppInitializer
@pytest.fixture()
def session() -> Iterator[Session]:
@pytest.fixture
def session(mocker: MockFixture) -> Iterator[Session]:
"""
Create an in-memory SQLite session to test models.
"""
@@ -40,11 +42,18 @@ def session() -> Iterator[Session]:
# flask calls session.remove()
in_memory_session.remove = lambda: None
# patch session
mocker.patch(
"superset.security.SupersetSecurityManager.get_session",
return_value=in_memory_session,
)
mocker.patch("superset.db.session", in_memory_session)
yield in_memory_session
@pytest.fixture
def app(mocker: MockFixture, session: Session) -> Iterator[SupersetApp]:
@pytest.fixture(scope="module")
def app() -> Iterator[SupersetApp]:
"""
A fixture that generates a Superset app.
"""
@@ -52,20 +61,31 @@ def app(mocker: MockFixture, session: Session) -> Iterator[SupersetApp]:
app.config.from_object("superset.config")
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite://"
app.config["FAB_ADD_SECURITY_VIEWS"] = False
app.config["TESTING"] = True
app_initializer = app.config.get("APP_INITIALIZER", SupersetAppInitializer)(app)
# ``superset.extensions.appbuilder`` is a singleton, and won't rebuild the
# routes when this fixture is called multiple times; we need to clear the
# registered views to ensure the initialization can happen more than once.
appbuilder.baseviews = []
app_initializer = SupersetAppInitializer(app)
app_initializer.init_app()
# patch session
mocker.patch(
"superset.security.SupersetSecurityManager.get_session", return_value=session,
)
mocker.patch("superset.db.session", session)
# reload base views to ensure error handlers are applied to the app
with app.app_context():
import superset.views.base
importlib.reload(superset.views.base)
yield app
@pytest.fixture
def client(app: SupersetApp) -> Any:
with app.test_client() as client:
yield client
@pytest.fixture
def app_context(app: SupersetApp) -> Iterator[None]:
"""

16
tests/unit_tests/importexport/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

254
tests/unit_tests/importexport/api_test.py Normal file
View File

@@ -0,0 +1,254 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=invalid-name, import-outside-toplevel
import json
from io import BytesIO
from pathlib import Path
from typing import Any
from zipfile import is_zipfile, ZipFile
from pytest_mock import MockFixture
from superset import security_manager
def test_export_assets(mocker: MockFixture, client: Any) -> None:
"""
Test exporting assets.
"""
from superset.commands.importers.v1.utils import get_contents_from_bundle
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
mocked_contents = [
(
"metadata.yaml",
"version: 1.0.0\ntype: assets\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
),
("databases/example.yaml", "<DATABASE CONTENTS>"),
]
ExportAssetsCommand = mocker.patch("superset.importexport.api.ExportAssetsCommand")
ExportAssetsCommand().run.return_value = mocked_contents[:]
response = client.get("/api/v1/assets/export/")
assert response.status_code == 200
buf = BytesIO(response.data)
assert is_zipfile(buf)
buf.seek(0)
with ZipFile(buf) as bundle:
contents = get_contents_from_bundle(bundle)
assert contents == dict(mocked_contents)
def test_import_assets(mocker: MockFixture, client: Any) -> None:
"""
Test importing assets.
"""
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
mocked_contents = {
"metadata.yaml": (
"version: 1.0.0\ntype: assets\ntimestamp: '2022-01-01T00:00:00+00:00'\n"
),
"databases/example.yaml": "<DATABASE CONTENTS>",
}
ImportAssetsCommand = mocker.patch("superset.importexport.api.ImportAssetsCommand")
root = Path("assets_export")
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
for path, contents in mocked_contents.items():
with bundle.open(str(root / path), "w") as fp:
fp.write(contents.encode())
buf.seek(0)
form_data = {
"bundle": (buf, "assets_export.zip"),
"passwords": json.dumps(
{"assets_export/databases/imported_database.yaml": "SECRET"}
),
}
response = client.post(
"/api/v1/assets/import/", data=form_data, content_type="multipart/form-data"
)
assert response.status_code == 200
assert response.json == {"message": "OK"}
passwords = {"assets_export/databases/imported_database.yaml": "SECRET"}
ImportAssetsCommand.assert_called_with(mocked_contents, passwords=passwords)
def test_import_assets_not_zip(mocker: MockFixture, client: Any) -> None:
"""
Test error message when the upload is not a ZIP file.
"""
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
buf = BytesIO(b"definitely_not_a_zip_file")
form_data = {
"bundle": (buf, "broken.txt"),
}
response = client.post(
"/api/v1/assets/import/", data=form_data, content_type="multipart/form-data"
)
assert response.status_code == 422
assert response.json == {
"errors": [
{
"message": "Not a ZIP file",
"error_type": "GENERIC_COMMAND_ERROR",
"level": "warning",
"extra": {
"issue_codes": [
{
"code": 1010,
"message": (
"Issue 1010 - Superset encountered an error while "
"running a command."
),
}
]
},
}
]
}
def test_import_assets_no_form_data(mocker: MockFixture, client: Any) -> None:
"""
Test error message when the upload has no form data.
"""
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
response = client.post("/api/v1/assets/import/", data="some_content")
assert response.status_code == 400
assert response.json == {
"errors": [
{
"message": "Request MIME type is not 'multipart/form-data'",
"error_type": "INVALID_PAYLOAD_FORMAT_ERROR",
"level": "error",
"extra": {
"issue_codes": [
{
"code": 1019,
"message": (
"Issue 1019 - The submitted payload has the incorrect "
"format."
),
}
]
},
}
]
}
def test_import_assets_incorrect_form_data(mocker: MockFixture, client: Any) -> None:
"""
Test error message when the upload form data has the wrong key.
"""
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
buf = BytesIO(b"definitely_not_a_zip_file")
form_data = {
"wrong": (buf, "broken.txt"),
}
response = client.post(
"/api/v1/assets/import/", data=form_data, content_type="multipart/form-data"
)
assert response.status_code == 400
assert response.json == {"message": "Arguments are not correct"}
def test_import_assets_no_contents(mocker: MockFixture, client: Any) -> None:
"""
Test error message when the ZIP bundle has no contents.
"""
# grant access
mocker.patch(
"flask_appbuilder.security.decorators.verify_jwt_in_request", return_value=True
)
mocker.patch.object(security_manager, "has_access", return_value=True)
mocked_contents = {
"README.txt": "Something is wrong",
}
root = Path("assets_export")
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
for path, contents in mocked_contents.items():
with bundle.open(str(root / path), "w") as fp:
fp.write(contents.encode())
buf.seek(0)
form_data = {
"bundle": (buf, "assets_export.zip"),
"passwords": json.dumps(
{"assets_export/databases/imported_database.yaml": "SECRET"}
),
}
response = client.post(
"/api/v1/assets/import/", data=form_data, content_type="multipart/form-data"
)
assert response.status_code == 400
assert response.json == {
"errors": [
{
"message": "No valid import files were found",
"error_type": "GENERIC_COMMAND_ERROR",
"level": "warning",
"extra": {
"issue_codes": [
{
"code": 1010,
"message": (
"Issue 1010 - Superset encountered an error while "
"running a command."
),
}
]
},
}
]
}

16
tests/unit_tests/views/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.