QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: add unit tests for the current import functionality (#11786)

Browse Source 
* chore: add unit tests for the current import functionality

* Improve comment

* Fix unit test
This commit is contained in:
Beto Dealmeida
2020-12-03 14:48:43 -08:00
committed by GitHub
parent 5b1939802d
commit e0288bf76b
5 changed files with 474 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
tests/dashboards/commands_tests.py
View File

@@ -25,14 +25,18 @@ import yaml
from superset import db, security_manager
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.exceptions import IncorrectVersionError
from superset.connectors.sqla.models import SqlaTable
from superset.dashboards.commands.exceptions import DashboardNotFoundError
from superset.dashboards.commands.export import ExportDashboardsCommand
from superset.dashboards.commands.importers.v1 import ImportDashboardsCommand
from superset.dashboards.commands.importers import v0, v1
from superset.models.core import Database
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from tests.base_tests import SupersetTestCase
from tests.fixtures.importexport import (
chart_config,
dashboard_config,
dashboard_export,
dashboard_metadata_config,
database_config,
dataset_config,
@@ -205,6 +209,45 @@ class TestExportDashboardsCommand(SupersetTestCase):
class TestImportDashboardsCommand(SupersetTestCase):
def test_import_v0_dashboard_cli_export(self):
num_dashboards = db.session.query(Dashboard).count()
num_charts = db.session.query(Slice).count()
num_datasets = db.session.query(SqlaTable).count()
num_databases = db.session.query(Database).count()
contents = {
"20201119_181105.json": json.dumps(dashboard_export),
}
command = v0.ImportDashboardsCommand(contents)
command.run()
new_num_dashboards = db.session.query(Dashboard).count()
new_num_charts = db.session.query(Slice).count()
new_num_datasets = db.session.query(SqlaTable).count()
new_num_databases = db.session.query(Database).count()
assert new_num_dashboards == num_dashboards + 1
assert new_num_charts == num_charts + 1
assert new_num_datasets == num_datasets + 1
assert new_num_databases == num_databases
dashboard = (
db.session.query(Dashboard).filter_by(dashboard_title="Births 2").one()
)
assert len(dashboard.slices) == 1
chart = dashboard.slices[0]
assert chart.slice_name == "Number of California Births"
dataset = chart.table
assert dataset.table_name == "birth_names_2"
database = dataset.database
assert database.database_name == "examples"
db.session.delete(dashboard)
db.session.delete(chart)
db.session.delete(dataset)
db.session.commit()
def test_import_v1_dashboard(self):
"""Test that we can import a dashboard"""
contents = {
@@ -214,7 +257,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
command.run()
dashboard = (
@@ -296,7 +339,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
command.run()
command.run()
@@ -325,7 +368,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
@@ -338,14 +381,14 @@ class TestImportDashboardsCommand(SupersetTestCase):
"timestamp": "2020-11-04T21:27:44.423819+00:00",
}
)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be Database
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dashboard"
@@ -358,7 +401,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
del broken_config["table_name"]
contents["metadata.yaml"] = yaml.safe_dump(dashboard_metadata_config)
contents["datasets/imported_dataset.yaml"] = yaml.safe_dump(broken_config)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dashboard"

98
tests/datasets/commands_tests.py
View File

@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=no-self-use, invalid-name
# pylint: disable=no-self-use, invalid-name, line-too-long
from operator import itemgetter
from unittest.mock import patch
@@ -29,15 +29,17 @@ from superset.connectors.sqla.models import SqlaTable
from superset.databases.commands.importers.v1 import ImportDatabasesCommand
from superset.datasets.commands.exceptions import DatasetNotFoundError
from superset.datasets.commands.export import ExportDatasetsCommand
from superset.datasets.commands.importers.v1 import ImportDatasetsCommand
from superset.datasets.commands.importers import v0, v1
from superset.models.core import Database
from superset.utils.core import get_example_database
from tests.base_tests import SupersetTestCase
from tests.fixtures.importexport import (
database_config,
database_metadata_config,
dataset_cli_export,
dataset_config,
dataset_metadata_config,
dataset_ui_export,
)
@@ -202,6 +204,78 @@ class TestExportDatasetsCommand(SupersetTestCase):
class TestImportDatasetsCommand(SupersetTestCase):
def test_import_v0_dataset_cli_export(self):
num_datasets = db.session.query(SqlaTable).count()
contents = {
"20201119_181105.yaml": yaml.safe_dump(dataset_cli_export),
}
command = v0.ImportDatasetsCommand(contents)
command.run()
new_num_datasets = db.session.query(SqlaTable).count()
assert new_num_datasets == num_datasets + 1
dataset = (
db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one()
)
assert (
dataset.params
== '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}'
)
assert len(dataset.metrics) == 2
assert dataset.main_dttm_col == "ds"
assert dataset.filter_select_enabled
assert [col.column_name for col in dataset.columns] == [
"num_california",
"ds",
"state",
"gender",
"name",
"sum_boys",
"sum_girls",
"num",
]
db.session.delete(dataset)
db.session.commit()
def test_import_v0_dataset_ui_export(self):
num_datasets = db.session.query(SqlaTable).count()
contents = {
"20201119_181105.yaml": yaml.safe_dump(dataset_ui_export),
}
command = v0.ImportDatasetsCommand(contents)
command.run()
new_num_datasets = db.session.query(SqlaTable).count()
assert new_num_datasets == num_datasets + 1
dataset = (
db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one()
)
assert (
dataset.params
== '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}'
)
assert len(dataset.metrics) == 2
assert dataset.main_dttm_col == "ds"
assert dataset.filter_select_enabled
assert [col.column_name for col in dataset.columns] == [
"num_california",
"ds",
"state",
"gender",
"name",
"sum_boys",
"sum_girls",
"num",
]
db.session.delete(dataset)
db.session.commit()
def test_import_v1_dataset(self):
"""Test that we can import a dataset"""
contents = {
@@ -209,7 +283,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
dataset = (
@@ -267,7 +341,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
command.run()
dataset = (
@@ -285,7 +359,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(new_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
dataset = (
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
@@ -305,7 +379,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
contents = {
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
@@ -318,14 +392,14 @@ class TestImportDatasetsCommand(SupersetTestCase):
"timestamp": "2020-11-04T21:27:44.423819+00:00",
}
)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be SqlaTable
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dataset"
@@ -338,7 +412,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(broken_config)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dataset"
@@ -369,10 +443,14 @@ class TestImportDatasetsCommand(SupersetTestCase):
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
"databases/imported_database.yaml": yaml.safe_dump(database_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert len(database.tables) == 1
db.session.delete(database.tables[0])
db.session.delete(database)
db.session.commit()

306
tests/fixtures/importexport.py vendored
View File

@@ -14,10 +14,312 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=line-too-long
from typing import Any, Dict
from typing import Any, Dict, List
# example YAML files
# example V0 import/export format
dataset_ui_export: List[Dict[str, Any]] = [
{
"columns": [
{
"column_name": "num_california",
"expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END",
},
{"column_name": "ds", "is_dttm": True, "type": "DATETIME"},
{"column_name": "state", "type": "VARCHAR(10)"},
{"column_name": "gender", "type": "VARCHAR(16)"},
{"column_name": "name", "type": "VARCHAR(255)"},
{"column_name": "sum_boys", "type": "BIGINT"},
{"column_name": "sum_girls", "type": "BIGINT"},
{"column_name": "num", "type": "BIGINT"},
],
"filter_select_enabled": True,
"main_dttm_col": "ds",
"metrics": [
{
"expression": "COUNT(*)",
"metric_name": "count",
"metric_type": "count",
"verbose_name": "COUNT(*)",
},
{"expression": "SUM(num)", "metric_name": "sum__num"},
],
"params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}',
"table_name": "birth_names_2",
}
]
dataset_cli_export: Dict[str, Any] = {
"databases": [
{
"allow_run_async": True,
"database_name": "examples",
"sqlalchemy_uri": "sqlite:////Users/beto/.superset/superset.db",
"tables": dataset_ui_export,
}
]
}
dashboard_export: Dict[str, Any] = {
"dashboards": [
{
"__Dashboard__": {
"css": "",
"dashboard_title": "Births 2",
"description": None,
"json_metadata": '{"timed_refresh_immune_slices": [], "expanded_slices": {}, "refresh_frequency": 0, "default_filters": "{}", "color_scheme": null, "remote_id": 1}',
"position_json": '{"CHART--jvaBFZx78":{"children":[],"id":"CHART--jvaBFZx78","meta":{"chartId":83,"height":50,"sliceName":"Number of California Births","uuid":"c77bb4b3-09f4-4d9a-a9e2-66a627c64343","width":4},"parents":["ROOT_ID","GRID_ID","ROW-se_5H8KNiO"],"type":"CHART"},"DASHBOARD_VERSION_KEY":"v2","GRID_ID":{"children":["ROW-se_5H8KNiO"],"id":"GRID_ID","parents":["ROOT_ID"],"type":"GRID"},"HEADER_ID":{"id":"HEADER_ID","meta":{"text":"Births"},"type":"HEADER"},"ROOT_ID":{"children":["GRID_ID"],"id":"ROOT_ID","type":"ROOT"},"ROW-se_5H8KNiO":{"children":["CHART--jvaBFZx78"],"id":"ROW-se_5H8KNiO","meta":{"background":"BACKGROUND_TRANSPARENT"},"parents":["ROOT_ID","GRID_ID"],"type":"ROW"}}',
"slices": [
{
"__Slice__": {
"cache_timeout": None,
"datasource_name": "birth_names_2",
"datasource_type": "table",
"id": 83,
"params": '{"adhoc_filters": [], "datasource": "3__table", "granularity_sqla": "ds", "header_font_size": 0.4, "metric": {"aggregate": "SUM", "column": {"column_name": "num_california", "expression": "CASE WHEN state = \'CA\' THEN num ELSE 0 END"}, "expressionType": "SIMPLE", "label": "SUM(num_california)"}, "queryFields": {"metric": "metrics"}, "slice_id": 83, "subheader_font_size": 0.15, "time_range": "100 years ago : now", "time_range_endpoints": ["unknown", "inclusive"], "url_params": {}, "viz_type": "big_number_total", "y_axis_format": "SMART_NUMBER", "remote_id": 83, "datasource_name": "birth_names_2", "schema": null, "database_name": "examples"}',
"slice_name": "Number of California Births",
"viz_type": "big_number_total",
}
}
],
"slug": None,
}
}
],
"datasources": [
{
"__SqlaTable__": {
"cache_timeout": None,
"columns": [
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "ds",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 332,
"is_active": True,
"is_dttm": True,
"python_date_format": None,
"table_id": 3,
"type": "DATETIME",
"uuid": "98e22f20-ed71-4483-b09d-31780ed1fc1b",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "gender",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 333,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(16)",
"uuid": "08e08f02-fb81-4461-bba6-c8c8dfef0c02",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "name",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(255)",
"uuid": "c67b14d9-fc4b-427d-a363-a53af015fb5e",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "num",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 335,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "69835b93-7169-4a2c-baa7-c1c92f21d10a",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "state",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 336,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(10)",
"uuid": "80003ad0-bdd0-48d3-ade3-8d1838e07d7a",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "sum_boys",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 337,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "8373ed24-4d4e-4307-9eee-8deefeecbb57",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "sum_girls",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 338,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "46f2de5f-c008-4024-a163-0b5c5f1d5580",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:32"},
"column_name": "num_california",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:32"},
"description": None,
"expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END",
"filterable": True,
"groupby": True,
"id": 434,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": None,
"uuid": "35e32aa6-be2b-4086-9c78-4ea3351ec079",
"verbose_name": None,
}
},
],
"database_id": 1000,
"default_endpoint": None,
"description": None,
"extra": None,
"fetch_values_predicate": None,
"filter_select_enabled": True,
"main_dttm_col": "ds",
"metrics": [
{
"__SqlMetric__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"d3format": None,
"description": None,
"expression": "COUNT(*)",
"extra": None,
"id": 9,
"metric_name": "count",
"metric_type": "count",
"table_id": 3,
"uuid": "1042ef50-ebf9-4271-b44e-3aaa891f6c21",
"verbose_name": "COUNT(*)",
"warning_text": None,
}
},
{
"__SqlMetric__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"d3format": None,
"description": None,
"expression": "SUM(num)",
"extra": None,
"id": 10,
"metric_name": "sum__num",
"metric_type": None,
"table_id": 3,
"uuid": "d807f208-e3c6-4b89-b790-41f521216ff6",
"verbose_name": None,
"warning_text": None,
}
},
],
"offset": 0,
"params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}',
"schema": None,
"sql": None,
"table_name": "birth_names_2",
"template_params": None,
}
}
],
}
# example V1 import/export format
database_metadata_config: Dict[str, Any] = {
"version": "1.0.0",
"type": "Database",