QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-05-11 19:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: set correct schema on config import (#16041)

Browse Source 
* fix: set correct schema on config import

* Fix lint

* Fix test

* Fix tests

* Fix another test

* Fix another test

* Fix base test

* Add helper function

* Fix examples

* Fix test

* Fix test

* Fixing more tests

(cherry picked from commit 1fbce88a46)
This commit is contained in:
Beto Dealmeida
2021-11-04 11:09:08 -07:00
committed by Elizabeth Thompson
parent 3d8ce130ba
commit 77c4f2cb11
30 changed files with 309 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tests/integration_tests/csv_upload_tests.py
View File

@@ -121,6 +121,7 @@ def get_upload_db():
def upload_csv(filename: str, table_name: str, extra: Optional[Dict[str, str]] = None):
csv_upload_db_id = get_upload_db().id
schema = utils.get_example_default_schema()
form_data = {
"csv_file": open(filename, "rb"),
"sep": ",",
@@ -130,6 +131,8 @@ def upload_csv(filename: str, table_name: str, extra: Optional[Dict[str, str]] =
"index_label": "test_label",
"mangle_dupe_cols": False,
}
if schema:
form_data["schema"] = schema
if extra:
form_data.update(extra)
return get_resp(test_client, "/csvtodatabaseview/form", data=form_data)
@@ -156,6 +159,7 @@ def upload_columnar(
filename: str, table_name: str, extra: Optional[Dict[str, str]] = None
):
columnar_upload_db_id = get_upload_db().id
schema = utils.get_example_default_schema()
form_data = {
"columnar_file": open(filename, "rb"),
"name": table_name,
@@ -163,6 +167,8 @@ def upload_columnar(
"if_exists": "fail",
"index_label": "test_label",
}
if schema:
form_data["schema"] = schema
if extra:
form_data.update(extra)
return get_resp(test_client, "/columnartodatabaseview/form", data=form_data)
@@ -208,7 +214,7 @@ def test_import_csv_enforced_schema(mock_event_logger):
full_table_name = f"admin_database.{CSV_UPLOAD_TABLE_W_SCHEMA}"
# no schema specified, fail upload
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE_W_SCHEMA)
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE_W_SCHEMA, extra={"schema": None})
assert (
f'Database "{CSV_UPLOAD_DATABASE}" schema "None" is not allowed for csv uploads'
in resp
@@ -256,14 +262,18 @@ def test_import_csv_enforced_schema(mock_event_logger):
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
def test_import_csv_explore_database(setup_csv_upload, create_csv_files):
schema = utils.get_example_default_schema()
full_table_name = (
f"{schema}.{CSV_UPLOAD_TABLE_W_EXPLORE}"
if schema
else CSV_UPLOAD_TABLE_W_EXPLORE
)
if utils.backend() == "sqlite":
pytest.skip("Sqlite doesn't support schema / database creation")
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE_W_EXPLORE)
assert (
f'CSV file "{CSV_FILENAME1}" uploaded to table "{CSV_UPLOAD_TABLE_W_EXPLORE}"'
in resp
)
assert f'CSV file "{CSV_FILENAME1}" uploaded to table "{full_table_name}"' in resp
table = SupersetTestCase.get_table(name=CSV_UPLOAD_TABLE_W_EXPLORE)
assert table.database_id == utils.get_example_database().id
@@ -273,9 +283,9 @@ def test_import_csv_explore_database(setup_csv_upload, create_csv_files):
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
@mock.patch("superset.views.database.views.event_logger.log_with_context")
def test_import_csv(mock_event_logger):
success_msg_f1 = (
f'CSV file "{CSV_FILENAME1}" uploaded to table "{CSV_UPLOAD_TABLE}"'
)
schema = utils.get_example_default_schema()
full_table_name = f"{schema}.{CSV_UPLOAD_TABLE}" if schema else CSV_UPLOAD_TABLE
success_msg_f1 = f'CSV file "{CSV_FILENAME1}" uploaded to table "{full_table_name}"'
test_db = get_upload_db()
@@ -299,7 +309,7 @@ def test_import_csv(mock_event_logger):
mock_event_logger.assert_called_with(
action="successful_csv_upload",
database=test_db.name,
schema=None,
schema=schema,
table=CSV_UPLOAD_TABLE,
)
@@ -328,9 +338,7 @@ def test_import_csv(mock_event_logger):
# replace table from file with different schema
resp = upload_csv(CSV_FILENAME2, CSV_UPLOAD_TABLE, extra={"if_exists": "replace"})
success_msg_f2 = (
f'CSV file "{CSV_FILENAME2}" uploaded to table "{CSV_UPLOAD_TABLE}"'
)
success_msg_f2 = f'CSV file "{CSV_FILENAME2}" uploaded to table "{full_table_name}"'
assert success_msg_f2 in resp
table = SupersetTestCase.get_table(name=CSV_UPLOAD_TABLE)
@@ -420,9 +428,13 @@ def test_import_parquet(mock_event_logger):
if utils.backend() == "hive":
pytest.skip("Hive doesn't allow parquet upload.")
schema = utils.get_example_default_schema()
full_table_name = (
f"{schema}.{PARQUET_UPLOAD_TABLE}" if schema else PARQUET_UPLOAD_TABLE
)
test_db = get_upload_db()
success_msg_f1 = f'Columnar file "[\'{PARQUET_FILENAME1}\']" uploaded to table "{PARQUET_UPLOAD_TABLE}"'
success_msg_f1 = f'Columnar file "[\'{PARQUET_FILENAME1}\']" uploaded to table "{full_table_name}"'
# initial upload with fail mode
resp = upload_columnar(PARQUET_FILENAME1, PARQUET_UPLOAD_TABLE)
@@ -442,7 +454,7 @@ def test_import_parquet(mock_event_logger):
mock_event_logger.assert_called_with(
action="successful_columnar_upload",
database=test_db.name,
schema=None,
schema=schema,
table=PARQUET_UPLOAD_TABLE,
)
@@ -455,7 +467,7 @@ def test_import_parquet(mock_event_logger):
assert success_msg_f1 in resp
# make sure only specified column name was read
table = SupersetTestCase.get_table(name=PARQUET_UPLOAD_TABLE)
table = SupersetTestCase.get_table(name=PARQUET_UPLOAD_TABLE, schema=None)
assert "b" not in table.column_names
# upload again with replace mode
@@ -475,7 +487,9 @@ def test_import_parquet(mock_event_logger):
resp = upload_columnar(
ZIP_FILENAME, PARQUET_UPLOAD_TABLE, extra={"if_exists": "replace"}
)
success_msg_f2 = f'Columnar file "[\'{ZIP_FILENAME}\']" uploaded to table "{PARQUET_UPLOAD_TABLE}"'
success_msg_f2 = (
f'Columnar file "[\'{ZIP_FILENAME}\']" uploaded to table "{full_table_name}"'
)
assert success_msg_f2 in resp
data = (