mirror of
https://github.com/apache/superset.git
synced 2026-04-19 08:04:53 +00:00
chore!: remove ENABLE_REACT_CRUD_VIEWS feature flag (permanently enable) (#19231)
* remove ENABLE_REACT_CRUD_VIEWS feature flag * docs * deal with problematic tests * empty test suite * skip test * test conditions changed * removing the tests instead of skipping
This commit is contained in:
David Aaron Suddjian
committed by
GitHub
GitHub
parent
4f0074a4ae
commit
97abc28a1f
@@ -203,318 +203,3 @@ def mock_upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str:
|
||||
container.exec_run(f"hdfs dfs -put {src} {dest}")
|
||||
# hive external table expectes a directory for the location
|
||||
return dest_dir
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_csv_upload")
|
||||
@pytest.mark.usefixtures("create_csv_files")
|
||||
@mock.patch(
|
||||
"superset.models.core.config",
|
||||
{**app.config, "ALLOWED_USER_CSV_SCHEMA_FUNC": lambda d, u: ["admin_database"]},
|
||||
)
|
||||
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
|
||||
@mock.patch("superset.views.database.views.event_logger.log_with_context")
|
||||
def test_import_csv_enforced_schema(mock_event_logger):
|
||||
if utils.backend() == "sqlite":
|
||||
pytest.skip("Sqlite doesn't support schema / database creation")
|
||||
|
||||
full_table_name = f"admin_database.{CSV_UPLOAD_TABLE_W_SCHEMA}"
|
||||
|
||||
# Invalid table name
|
||||
resp = upload_csv(CSV_FILENAME1, full_table_name)
|
||||
assert "Table name cannot contain a schema" in resp
|
||||
|
||||
# no schema specified, fail upload
|
||||
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE_W_SCHEMA, extra={"schema": None})
|
||||
assert (
|
||||
f'Database "{CSV_UPLOAD_DATABASE}" schema "None" is not allowed for csv uploads'
|
||||
in resp
|
||||
)
|
||||
|
||||
success_msg = f'CSV file "{CSV_FILENAME1}" uploaded to table "{full_table_name}"'
|
||||
resp = upload_csv(
|
||||
CSV_FILENAME1,
|
||||
CSV_UPLOAD_TABLE_W_SCHEMA,
|
||||
extra={"schema": "admin_database", "if_exists": "replace"},
|
||||
)
|
||||
assert success_msg in resp
|
||||
mock_event_logger.assert_called_with(
|
||||
action="successful_csv_upload",
|
||||
database=get_upload_db().name,
|
||||
schema="admin_database",
|
||||
table=CSV_UPLOAD_TABLE_W_SCHEMA,
|
||||
)
|
||||
|
||||
engine = get_upload_db().get_sqla_engine()
|
||||
data = engine.execute(
|
||||
f"SELECT * from {ADMIN_SCHEMA_NAME}.{CSV_UPLOAD_TABLE_W_SCHEMA}"
|
||||
).fetchall()
|
||||
assert data == [("john", 1), ("paul", 2)]
|
||||
|
||||
# user specified schema doesn't match, fail
|
||||
resp = upload_csv(
|
||||
CSV_FILENAME1, CSV_UPLOAD_TABLE_W_SCHEMA, extra={"schema": "gold"}
|
||||
)
|
||||
assert (
|
||||
f'Database "{CSV_UPLOAD_DATABASE}" schema "gold" is not allowed for csv uploads'
|
||||
in resp
|
||||
)
|
||||
|
||||
# user specified schema matches the expected schema, append
|
||||
if utils.backend() == "hive":
|
||||
pytest.skip("Hive database doesn't support append csv uploads.")
|
||||
resp = upload_csv(
|
||||
CSV_FILENAME1,
|
||||
CSV_UPLOAD_TABLE_W_SCHEMA,
|
||||
extra={"schema": "admin_database", "if_exists": "append"},
|
||||
)
|
||||
assert success_msg in resp
|
||||
|
||||
|
||||
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
|
||||
def test_import_csv_explore_database(setup_csv_upload, create_csv_files):
|
||||
schema = utils.get_example_default_schema()
|
||||
full_table_name = (
|
||||
f"{schema}.{CSV_UPLOAD_TABLE_W_EXPLORE}"
|
||||
if schema
|
||||
else CSV_UPLOAD_TABLE_W_EXPLORE
|
||||
)
|
||||
|
||||
if utils.backend() == "sqlite":
|
||||
pytest.skip("Sqlite doesn't support schema / database creation")
|
||||
|
||||
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE_W_EXPLORE)
|
||||
assert f'CSV file "{CSV_FILENAME1}" uploaded to table "{full_table_name}"' in resp
|
||||
table = SupersetTestCase.get_table(name=CSV_UPLOAD_TABLE_W_EXPLORE)
|
||||
assert table.database_id == superset.utils.database.get_example_database().id
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_csv_upload")
|
||||
@pytest.mark.usefixtures("create_csv_files")
|
||||
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
|
||||
@mock.patch("superset.views.database.views.event_logger.log_with_context")
|
||||
def test_import_csv(mock_event_logger):
|
||||
schema = utils.get_example_default_schema()
|
||||
full_table_name = f"{schema}.{CSV_UPLOAD_TABLE}" if schema else CSV_UPLOAD_TABLE
|
||||
success_msg_f1 = f'CSV file "{CSV_FILENAME1}" uploaded to table "{full_table_name}"'
|
||||
|
||||
test_db = get_upload_db()
|
||||
|
||||
# initial upload with fail mode
|
||||
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE)
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
# upload again with fail mode; should fail
|
||||
fail_msg = (
|
||||
f'Unable to upload CSV file "{CSV_FILENAME1}" to table "{CSV_UPLOAD_TABLE}"'
|
||||
)
|
||||
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE)
|
||||
assert fail_msg in resp
|
||||
|
||||
if utils.backend() != "hive":
|
||||
# upload again with append mode
|
||||
resp = upload_csv(
|
||||
CSV_FILENAME1, CSV_UPLOAD_TABLE, extra={"if_exists": "append"}
|
||||
)
|
||||
assert success_msg_f1 in resp
|
||||
mock_event_logger.assert_called_with(
|
||||
action="successful_csv_upload",
|
||||
database=test_db.name,
|
||||
schema=schema,
|
||||
table=CSV_UPLOAD_TABLE,
|
||||
)
|
||||
|
||||
# upload again with replace mode and specific columns
|
||||
resp = upload_csv(
|
||||
CSV_FILENAME1,
|
||||
CSV_UPLOAD_TABLE,
|
||||
extra={"if_exists": "replace", "usecols": '["a"]'},
|
||||
)
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
# make sure only specified column name was read
|
||||
table = SupersetTestCase.get_table(name=CSV_UPLOAD_TABLE)
|
||||
assert "b" not in table.column_names
|
||||
|
||||
# upload again with replace mode
|
||||
resp = upload_csv(CSV_FILENAME1, CSV_UPLOAD_TABLE, extra={"if_exists": "replace"})
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
# try to append to table from file with different schema
|
||||
resp = upload_csv(CSV_FILENAME2, CSV_UPLOAD_TABLE, extra={"if_exists": "append"})
|
||||
fail_msg_f2 = (
|
||||
f'Unable to upload CSV file "{CSV_FILENAME2}" to table "{CSV_UPLOAD_TABLE}"'
|
||||
)
|
||||
assert fail_msg_f2 in resp
|
||||
|
||||
# replace table from file with different schema
|
||||
resp = upload_csv(CSV_FILENAME2, CSV_UPLOAD_TABLE, extra={"if_exists": "replace"})
|
||||
success_msg_f2 = f'CSV file "{CSV_FILENAME2}" uploaded to table "{full_table_name}"'
|
||||
assert success_msg_f2 in resp
|
||||
|
||||
table = SupersetTestCase.get_table(name=CSV_UPLOAD_TABLE)
|
||||
# make sure the new column name is reflected in the table metadata
|
||||
assert "d" in table.column_names
|
||||
|
||||
# ensure user is assigned as an owner
|
||||
assert security_manager.find_user("admin") in table.owners
|
||||
|
||||
# null values are set
|
||||
upload_csv(
|
||||
CSV_FILENAME2,
|
||||
CSV_UPLOAD_TABLE,
|
||||
extra={"null_values": '["", "john"]', "if_exists": "replace"},
|
||||
)
|
||||
# make sure that john and empty string are replaced with None
|
||||
engine = test_db.get_sqla_engine()
|
||||
data = engine.execute(f"SELECT * from {CSV_UPLOAD_TABLE}").fetchall()
|
||||
assert data == [(None, 1, "x"), ("paul", 2, None)]
|
||||
|
||||
# default null values
|
||||
upload_csv(CSV_FILENAME2, CSV_UPLOAD_TABLE, extra={"if_exists": "replace"})
|
||||
# make sure that john and empty string are replaced with None
|
||||
data = engine.execute(f"SELECT * from {CSV_UPLOAD_TABLE}").fetchall()
|
||||
assert data == [("john", 1, "x"), ("paul", 2, None)]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_csv_upload")
|
||||
@pytest.mark.usefixtures("create_excel_files")
|
||||
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
|
||||
@mock.patch("superset.views.database.views.event_logger.log_with_context")
|
||||
def test_import_excel(mock_event_logger):
|
||||
if utils.backend() == "hive":
|
||||
pytest.skip("Hive doesn't excel upload.")
|
||||
|
||||
schema = utils.get_example_default_schema()
|
||||
full_table_name = f"{schema}.{EXCEL_UPLOAD_TABLE}" if schema else EXCEL_UPLOAD_TABLE
|
||||
test_db = get_upload_db()
|
||||
|
||||
success_msg = f'Excel file "{EXCEL_FILENAME}" uploaded to table "{full_table_name}"'
|
||||
|
||||
# initial upload with fail mode
|
||||
resp = upload_excel(EXCEL_FILENAME, EXCEL_UPLOAD_TABLE)
|
||||
assert success_msg in resp
|
||||
mock_event_logger.assert_called_with(
|
||||
action="successful_excel_upload",
|
||||
database=test_db.name,
|
||||
schema=schema,
|
||||
table=EXCEL_UPLOAD_TABLE,
|
||||
)
|
||||
|
||||
# ensure user is assigned as an owner
|
||||
table = SupersetTestCase.get_table(name=EXCEL_UPLOAD_TABLE)
|
||||
assert security_manager.find_user("admin") in table.owners
|
||||
|
||||
# upload again with fail mode; should fail
|
||||
fail_msg = f'Unable to upload Excel file "{EXCEL_FILENAME}" to table "{EXCEL_UPLOAD_TABLE}"'
|
||||
resp = upload_excel(EXCEL_FILENAME, EXCEL_UPLOAD_TABLE)
|
||||
assert fail_msg in resp
|
||||
|
||||
if utils.backend() != "hive":
|
||||
# upload again with append mode
|
||||
resp = upload_excel(
|
||||
EXCEL_FILENAME, EXCEL_UPLOAD_TABLE, extra={"if_exists": "append"}
|
||||
)
|
||||
assert success_msg in resp
|
||||
|
||||
# upload again with replace mode
|
||||
resp = upload_excel(
|
||||
EXCEL_FILENAME, EXCEL_UPLOAD_TABLE, extra={"if_exists": "replace"}
|
||||
)
|
||||
assert success_msg in resp
|
||||
mock_event_logger.assert_called_with(
|
||||
action="successful_excel_upload",
|
||||
database=test_db.name,
|
||||
schema=schema,
|
||||
table=EXCEL_UPLOAD_TABLE,
|
||||
)
|
||||
|
||||
# make sure that john and empty string are replaced with None
|
||||
data = (
|
||||
test_db.get_sqla_engine()
|
||||
.execute(f"SELECT * from {EXCEL_UPLOAD_TABLE}")
|
||||
.fetchall()
|
||||
)
|
||||
assert data == [(0, "john", 1), (1, "paul", 2)]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_csv_upload")
|
||||
@pytest.mark.usefixtures("create_columnar_files")
|
||||
@mock.patch("superset.db_engine_specs.hive.upload_to_s3", mock_upload_to_s3)
|
||||
@mock.patch("superset.views.database.views.event_logger.log_with_context")
|
||||
def test_import_parquet(mock_event_logger):
|
||||
if utils.backend() == "hive":
|
||||
pytest.skip("Hive doesn't allow parquet upload.")
|
||||
|
||||
schema = utils.get_example_default_schema()
|
||||
full_table_name = (
|
||||
f"{schema}.{PARQUET_UPLOAD_TABLE}" if schema else PARQUET_UPLOAD_TABLE
|
||||
)
|
||||
test_db = get_upload_db()
|
||||
|
||||
success_msg_f1 = f'Columnar file "[\'{PARQUET_FILENAME1}\']" uploaded to table "{full_table_name}"'
|
||||
|
||||
# initial upload with fail mode
|
||||
resp = upload_columnar(PARQUET_FILENAME1, PARQUET_UPLOAD_TABLE)
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
# upload again with fail mode; should fail
|
||||
fail_msg = f'Unable to upload Columnar file "[\'{PARQUET_FILENAME1}\']" to table "{PARQUET_UPLOAD_TABLE}"'
|
||||
resp = upload_columnar(PARQUET_FILENAME1, PARQUET_UPLOAD_TABLE)
|
||||
assert fail_msg in resp
|
||||
|
||||
if utils.backend() != "hive":
|
||||
# upload again with append mode
|
||||
resp = upload_columnar(
|
||||
PARQUET_FILENAME1, PARQUET_UPLOAD_TABLE, extra={"if_exists": "append"}
|
||||
)
|
||||
assert success_msg_f1 in resp
|
||||
mock_event_logger.assert_called_with(
|
||||
action="successful_columnar_upload",
|
||||
database=test_db.name,
|
||||
schema=schema,
|
||||
table=PARQUET_UPLOAD_TABLE,
|
||||
)
|
||||
|
||||
# upload again with replace mode and specific columns
|
||||
resp = upload_columnar(
|
||||
PARQUET_FILENAME1,
|
||||
PARQUET_UPLOAD_TABLE,
|
||||
extra={"if_exists": "replace", "usecols": '["a"]'},
|
||||
)
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
table = SupersetTestCase.get_table(name=PARQUET_UPLOAD_TABLE, schema=None)
|
||||
# make sure only specified column name was read
|
||||
assert "b" not in table.column_names
|
||||
|
||||
# ensure user is assigned as an owner
|
||||
assert security_manager.find_user("admin") in table.owners
|
||||
|
||||
# upload again with replace mode
|
||||
resp = upload_columnar(
|
||||
PARQUET_FILENAME1, PARQUET_UPLOAD_TABLE, extra={"if_exists": "replace"}
|
||||
)
|
||||
assert success_msg_f1 in resp
|
||||
|
||||
data = (
|
||||
test_db.get_sqla_engine()
|
||||
.execute(f"SELECT * from {PARQUET_UPLOAD_TABLE} ORDER BY b")
|
||||
.fetchall()
|
||||
)
|
||||
assert data == [("john", 1), ("paul", 2)]
|
||||
|
||||
# replace table with zip file
|
||||
resp = upload_columnar(
|
||||
ZIP_FILENAME, PARQUET_UPLOAD_TABLE, extra={"if_exists": "replace"}
|
||||
)
|
||||
success_msg_f2 = (
|
||||
f'Columnar file "[\'{ZIP_FILENAME}\']" uploaded to table "{full_table_name}"'
|
||||
)
|
||||
assert success_msg_f2 in resp
|
||||
|
||||
data = (
|
||||
test_db.get_sqla_engine()
|
||||
.execute(f"SELECT * from {PARQUET_UPLOAD_TABLE} ORDER BY b")
|
||||
.fetchall()
|
||||
)
|
||||
assert data == [("john", 1), ("paul", 2), ("max", 3), ("bob", 4)]
|
||||
|
||||
Reference in New Issue
Block a user