mirror of
https://github.com/apache/superset.git
synced 2026-04-13 21:24:28 +00:00
refactor: upload data unification, less permissions and less endpoints (#31959)
This commit is contained in:
Daniel Vaz Gaspar
committed by
GitHub
GitHub
parent
09c1987de4
commit
1b375b715c
@@ -842,6 +842,7 @@ def test_oauth2_error(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -855,6 +856,7 @@ def test_oauth2_error(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"already_exists": "fail",
|
||||
"delimiter": ",",
|
||||
"file": ANY,
|
||||
@@ -864,6 +866,7 @@ def test_oauth2_error(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table2",
|
||||
"delimiter": ";",
|
||||
@@ -879,6 +882,7 @@ def test_oauth2_error(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"already_exists": "replace",
|
||||
"column_dates": ["col1", "col2"],
|
||||
"delimiter": ";",
|
||||
@@ -889,6 +893,7 @@ def test_oauth2_error(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table2",
|
||||
"delimiter": ";",
|
||||
@@ -911,6 +916,7 @@ def test_oauth2_error(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"already_exists": "replace",
|
||||
"columns_read": ["col1", "col2"],
|
||||
"null_values": ["None", "N/A", "''"],
|
||||
@@ -945,7 +951,7 @@ def test_csv_upload(
|
||||
reader_mock = mocker.patch.object(CSVReader, "__init__")
|
||||
reader_mock.return_value = None
|
||||
response = client.post(
|
||||
"/api/v1/database/1/csv_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -960,6 +966,7 @@ def test_csv_upload(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"delimiter": ",",
|
||||
"already_exists": "fail",
|
||||
@@ -968,6 +975,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "",
|
||||
"delimiter": ",",
|
||||
@@ -976,11 +984,17 @@ def test_csv_upload(
|
||||
{"message": {"table_name": ["Length must be between 1 and 10000."]}},
|
||||
),
|
||||
(
|
||||
{"table_name": "table1", "delimiter": ",", "already_exists": "fail"},
|
||||
{
|
||||
"type": "csv",
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
"already_exists": "fail",
|
||||
},
|
||||
{"message": {"file": ["Field may not be null."]}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": "xpto",
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -990,6 +1004,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -999,6 +1014,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1009,6 +1025,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1019,6 +1036,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1029,6 +1047,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1039,6 +1058,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1049,6 +1069,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1059,6 +1080,7 @@ def test_csv_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "csv",
|
||||
"file": (create_csv_file(), "out.csv"),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1082,7 +1104,7 @@ def test_csv_upload_validation(
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/database/1/csv_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -1116,8 +1138,9 @@ def test_csv_upload_file_extension_invalid(
|
||||
"""
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
response = client.post(
|
||||
"/api/v1/database/1/csv_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data={
|
||||
"type": "csv",
|
||||
"file": create_csv_file(filename=filename),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1152,8 +1175,9 @@ def test_csv_upload_file_extension_valid(
|
||||
"""
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
response = client.post(
|
||||
"/api/v1/database/1/csv_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data={
|
||||
"type": "csv",
|
||||
"file": create_csv_file(filename=filename),
|
||||
"table_name": "table1",
|
||||
"delimiter": ",",
|
||||
@@ -1168,6 +1192,7 @@ def test_csv_upload_file_extension_valid(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table1",
|
||||
},
|
||||
@@ -1180,6 +1205,7 @@ def test_csv_upload_file_extension_valid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"already_exists": "fail",
|
||||
"file": ANY,
|
||||
"table_name": "table1",
|
||||
@@ -1188,6 +1214,7 @@ def test_csv_upload_file_extension_valid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table2",
|
||||
"sheet_name": "Sheet1",
|
||||
@@ -1203,6 +1230,7 @@ def test_csv_upload_file_extension_valid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"already_exists": "replace",
|
||||
"column_dates": ["col1", "col2"],
|
||||
"sheet_name": "Sheet1",
|
||||
@@ -1213,6 +1241,7 @@ def test_csv_upload_file_extension_valid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table2",
|
||||
"sheet_name": "Sheet1",
|
||||
@@ -1231,6 +1260,7 @@ def test_csv_upload_file_extension_valid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"already_exists": "replace",
|
||||
"columns_read": ["col1", "col2"],
|
||||
"null_values": ["None", "N/A", "''"],
|
||||
@@ -1261,7 +1291,7 @@ def test_excel_upload(
|
||||
reader_mock = mocker.patch.object(ExcelReader, "__init__")
|
||||
reader_mock.return_value = None
|
||||
response = client.post(
|
||||
"/api/v1/database/1/excel_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -1276,6 +1306,7 @@ def test_excel_upload(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"sheet_name": "Sheet1",
|
||||
"already_exists": "fail",
|
||||
@@ -1284,6 +1315,7 @@ def test_excel_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "",
|
||||
"sheet_name": "Sheet1",
|
||||
@@ -1292,11 +1324,12 @@ def test_excel_upload(
|
||||
{"message": {"table_name": ["Length must be between 1 and 10000."]}},
|
||||
),
|
||||
(
|
||||
{"table_name": "table1", "already_exists": "fail"},
|
||||
{"type": "excel", "table_name": "table1", "already_exists": "fail"},
|
||||
{"message": {"file": ["Field may not be null."]}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": "xpto",
|
||||
"table_name": "table1",
|
||||
"already_exists": "fail",
|
||||
@@ -1305,6 +1338,7 @@ def test_excel_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table1",
|
||||
"already_exists": "xpto",
|
||||
@@ -1313,6 +1347,7 @@ def test_excel_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table1",
|
||||
"already_exists": "fail",
|
||||
@@ -1322,6 +1357,7 @@ def test_excel_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table1",
|
||||
"already_exists": "fail",
|
||||
@@ -1331,6 +1367,7 @@ def test_excel_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "excel",
|
||||
"file": (create_excel_file(), "out.xls"),
|
||||
"table_name": "table1",
|
||||
"already_exists": "fail",
|
||||
@@ -1353,7 +1390,7 @@ def test_excel_upload_validation(
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/database/1/excel_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -1387,8 +1424,9 @@ def test_excel_upload_file_extension_invalid(
|
||||
"""
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
response = client.post(
|
||||
"/api/v1/database/1/excel_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data={
|
||||
"type": "excel",
|
||||
"file": create_excel_file(filename=filename),
|
||||
"table_name": "table1",
|
||||
},
|
||||
@@ -1403,6 +1441,7 @@ def test_excel_upload_file_extension_invalid(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), "out.parquet"),
|
||||
"table_name": "table1",
|
||||
},
|
||||
@@ -1415,6 +1454,7 @@ def test_excel_upload_file_extension_invalid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"already_exists": "fail",
|
||||
"file": ANY,
|
||||
"table_name": "table1",
|
||||
@@ -1423,6 +1463,7 @@ def test_excel_upload_file_extension_invalid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), "out.parquet"),
|
||||
"table_name": "table2",
|
||||
"already_exists": "replace",
|
||||
@@ -1439,6 +1480,7 @@ def test_excel_upload_file_extension_invalid(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"already_exists": "replace",
|
||||
"columns_read": ["col1", "col2"],
|
||||
"file": ANY,
|
||||
@@ -1467,7 +1509,7 @@ def test_columnar_upload(
|
||||
reader_mock = mocker.patch.object(ColumnarReader, "__init__")
|
||||
reader_mock.return_value = None
|
||||
response = client.post(
|
||||
"/api/v1/database/1/columnar_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -1482,6 +1524,7 @@ def test_columnar_upload(
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), "out.parquet"),
|
||||
"already_exists": "fail",
|
||||
},
|
||||
@@ -1489,6 +1532,7 @@ def test_columnar_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), "out.parquet"),
|
||||
"table_name": "",
|
||||
"already_exists": "fail",
|
||||
@@ -1496,11 +1540,12 @@ def test_columnar_upload(
|
||||
{"message": {"table_name": ["Length must be between 1 and 10000."]}},
|
||||
),
|
||||
(
|
||||
{"table_name": "table1", "already_exists": "fail"},
|
||||
{"type": "columnar", "table_name": "table1", "already_exists": "fail"},
|
||||
{"message": {"file": ["Field may not be null."]}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": "xpto",
|
||||
"table_name": "table1",
|
||||
"already_exists": "fail",
|
||||
@@ -1509,6 +1554,7 @@ def test_columnar_upload(
|
||||
),
|
||||
(
|
||||
{
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), "out.parquet"),
|
||||
"table_name": "table1",
|
||||
"already_exists": "xpto",
|
||||
@@ -1530,7 +1576,7 @@ def test_columnar_upload_validation(
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/database/1/columnar_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data=payload,
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
@@ -1559,8 +1605,9 @@ def test_columnar_upload_file_extension_valid(
|
||||
"""
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
response = client.post(
|
||||
"/api/v1/database/1/columnar_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data={
|
||||
"type": "columnar",
|
||||
"file": (create_columnar_file(), filename),
|
||||
"table_name": "table1",
|
||||
},
|
||||
@@ -1595,8 +1642,9 @@ def test_columnar_upload_file_extension_invalid(
|
||||
"""
|
||||
_ = mocker.patch.object(UploadCommand, "run")
|
||||
response = client.post(
|
||||
"/api/v1/database/1/columnar_upload/",
|
||||
"/api/v1/database/1/upload/",
|
||||
data={
|
||||
"type": "columnar",
|
||||
"file": create_columnar_file(filename=filename),
|
||||
"table_name": "table1",
|
||||
},
|
||||
@@ -1611,8 +1659,8 @@ def test_csv_metadata(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(CSVReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/csv_metadata/",
|
||||
data={"file": create_csv_file()},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "csv", "file": create_csv_file()},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
@@ -1623,8 +1671,8 @@ def test_csv_metadata_bad_extension(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(CSVReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/csv_metadata/",
|
||||
data={"file": create_csv_file(filename="test.out")},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "csv", "file": create_csv_file(filename="test.out")},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
@@ -1636,21 +1684,29 @@ def test_csv_metadata_validation(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(CSVReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/csv_metadata/",
|
||||
data={},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "csv"},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json == {"message": {"file": ["Field may not be null."]}}
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"file": create_csv_file(filename="test.csv")},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json == {"message": {"type": ["Missing data for required field."]}}
|
||||
|
||||
|
||||
def test_excel_metadata(
|
||||
mocker: MockerFixture, client: Any, full_api_access: None
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ExcelReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/excel_metadata/",
|
||||
data={"file": create_excel_file()},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "excel", "file": create_excel_file()},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
@@ -1661,8 +1717,8 @@ def test_excel_metadata_bad_extension(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ExcelReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/excel_metadata/",
|
||||
data={"file": create_excel_file(filename="test.out")},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "excel", "file": create_excel_file(filename="test.out")},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
@@ -1674,8 +1730,8 @@ def test_excel_metadata_validation(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ExcelReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/excel_metadata/",
|
||||
data={},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "excel"},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
@@ -1687,8 +1743,8 @@ def test_columnar_metadata(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ColumnarReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/columnar_metadata/",
|
||||
data={"file": create_columnar_file()},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "columnar", "file": create_columnar_file()},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
@@ -1699,8 +1755,8 @@ def test_columnar_metadata_bad_extension(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ColumnarReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/columnar_metadata/",
|
||||
data={"file": create_columnar_file(filename="test.out")},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "columnar", "file": create_columnar_file(filename="test.out")},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
@@ -1712,8 +1768,8 @@ def test_columnar_metadata_validation(
|
||||
) -> None:
|
||||
_ = mocker.patch.object(ColumnarReader, "file_metadata")
|
||||
response = client.post(
|
||||
"/api/v1/database/columnar_metadata/",
|
||||
data={},
|
||||
"/api/v1/database/upload_metadata/",
|
||||
data={"type": "columnar"},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
Reference in New Issue
Block a user