mirror of
https://github.com/apache/superset.git
synced 2026-04-19 08:04:53 +00:00
feat: new Columnar upload form and API (#28192)
This commit is contained in:
Daniel Vaz Gaspar
committed by
GitHub
GitHub
parent
f5843fe588
commit
9a339f08a7
@@ -58,6 +58,7 @@ from superset.commands.database.tables import TablesDatabaseCommand
|
||||
from superset.commands.database.test_connection import TestConnectionDatabaseCommand
|
||||
from superset.commands.database.update import UpdateDatabaseCommand
|
||||
from superset.commands.database.uploaders.base import UploadCommand
|
||||
from superset.commands.database.uploaders.columnar_reader import ColumnarReader
|
||||
from superset.commands.database.uploaders.csv_reader import CSVReader
|
||||
from superset.commands.database.uploaders.excel_reader import ExcelReader
|
||||
from superset.commands.database.validate import ValidateDatabaseParametersCommand
|
||||
@@ -72,6 +73,9 @@ from superset.daos.database import DatabaseDAO, DatabaseUserOAuth2TokensDAO
|
||||
from superset.databases.decorators import check_table_access
|
||||
from superset.databases.filters import DatabaseFilter, DatabaseUploadEnabledFilter
|
||||
from superset.databases.schemas import (
|
||||
ColumnarMetadataUploadFilePostSchema,
|
||||
ColumnarUploadPostSchema,
|
||||
CSVMetadataUploadFilePostSchema,
|
||||
CSVUploadPostSchema,
|
||||
database_schemas_query_schema,
|
||||
database_tables_query_schema,
|
||||
@@ -84,6 +88,7 @@ from superset.databases.schemas import (
|
||||
DatabaseTablesResponse,
|
||||
DatabaseTestConnectionSchema,
|
||||
DatabaseValidateParametersSchema,
|
||||
ExcelMetadataUploadFilePostSchema,
|
||||
ExcelUploadPostSchema,
|
||||
get_export_ids_schema,
|
||||
OAuth2ProviderResponseSchema,
|
||||
@@ -93,6 +98,7 @@ from superset.databases.schemas import (
|
||||
SelectStarResponseSchema,
|
||||
TableExtraMetadataResponseSchema,
|
||||
TableMetadataResponseSchema,
|
||||
UploadFileMetadata,
|
||||
ValidateSQLRequest,
|
||||
ValidateSQLResponse,
|
||||
)
|
||||
@@ -151,7 +157,11 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
"schemas_access_for_file_upload",
|
||||
"get_connection",
|
||||
"csv_upload",
|
||||
"csv_metadata",
|
||||
"excel_upload",
|
||||
"excel_metadata",
|
||||
"columnar_upload",
|
||||
"columnar_metadata",
|
||||
"oauth2",
|
||||
}
|
||||
|
||||
@@ -263,6 +273,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
|
||||
openapi_spec_tag = "Database"
|
||||
openapi_spec_component_schemas = (
|
||||
ColumnarUploadPostSchema,
|
||||
CSVUploadPostSchema,
|
||||
DatabaseConnectionSchema,
|
||||
DatabaseFunctionNamesResponse,
|
||||
@@ -276,6 +287,10 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
TableMetadataResponseSchema,
|
||||
SelectStarResponseSchema,
|
||||
SchemasResponseSchema,
|
||||
CSVMetadataUploadFilePostSchema,
|
||||
ExcelMetadataUploadFilePostSchema,
|
||||
ColumnarMetadataUploadFilePostSchema,
|
||||
UploadFileMetadata,
|
||||
ValidateSQLRequest,
|
||||
ValidateSQLResponse,
|
||||
)
|
||||
@@ -1524,11 +1539,60 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
command.run()
|
||||
return self.response(200, message="OK")
|
||||
|
||||
@expose("/csv_metadata/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=(
|
||||
lambda self, *args, **kwargs: f"{self.__class__.__name__}" ".csv_metadata"
|
||||
),
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
def csv_metadata(self) -> Response:
|
||||
"""Upload an CSV file and returns file metadata.
|
||||
---
|
||||
post:
|
||||
summary: Upload an CSV file and returns file metadata
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/CSVMetadataUploadFilePostSchema'
|
||||
responses:
|
||||
200:
|
||||
description: Columnar upload response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
result:
|
||||
$ref: '#/components/schemas/UploadFileMetadata'
|
||||
400:
|
||||
$ref: '#/components/responses/400'
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
404:
|
||||
$ref: '#/components/responses/404'
|
||||
500:
|
||||
$ref: '#/components/responses/500'
|
||||
"""
|
||||
try:
|
||||
request_form = request.form.to_dict()
|
||||
request_form["file"] = request.files.get("file")
|
||||
parameters = CSVMetadataUploadFilePostSchema().load(request_form)
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
metadata = CSVReader(parameters).file_metadata(parameters["file"])
|
||||
return self.response(200, result=UploadFileMetadata().dump(metadata))
|
||||
|
||||
@expose("/<int:pk>/csv_upload/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.import_",
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.csv_upload",
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
@@ -1549,7 +1613,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
schema:
|
||||
$ref: '#/components/schemas/CSVUploadPostSchema'
|
||||
responses:
|
||||
200:
|
||||
201:
|
||||
description: CSV upload response
|
||||
content:
|
||||
application/json:
|
||||
@@ -1582,13 +1646,62 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
).run()
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
return self.response(200, message="OK")
|
||||
return self.response(201, message="OK")
|
||||
|
||||
@expose("/excel_metadata/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=(
|
||||
lambda self, *args, **kwargs: f"{self.__class__.__name__}" ".excel_metadata"
|
||||
),
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
def excel_metadata(self) -> Response:
|
||||
"""Upload an Excel file and returns file metadata.
|
||||
---
|
||||
post:
|
||||
summary: Upload an Excel file and returns file metadata
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ExcelMetadataUploadFilePostSchema'
|
||||
responses:
|
||||
200:
|
||||
description: Columnar upload response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
result:
|
||||
$ref: '#/components/schemas/UploadFileMetadata'
|
||||
400:
|
||||
$ref: '#/components/responses/400'
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
404:
|
||||
$ref: '#/components/responses/404'
|
||||
500:
|
||||
$ref: '#/components/responses/500'
|
||||
"""
|
||||
try:
|
||||
request_form = request.form.to_dict()
|
||||
request_form["file"] = request.files.get("file")
|
||||
parameters = ExcelMetadataUploadFilePostSchema().load(request_form)
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
metadata = ExcelReader().file_metadata(parameters["file"])
|
||||
return self.response(200, result=UploadFileMetadata().dump(metadata))
|
||||
|
||||
@expose("/<int:pk>/excel_upload/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.import_",
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.excel_upload",
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
@@ -1609,7 +1722,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
schema:
|
||||
$ref: '#/components/schemas/ExcelUploadPostSchema'
|
||||
responses:
|
||||
200:
|
||||
201:
|
||||
description: Excel upload response
|
||||
content:
|
||||
application/json:
|
||||
@@ -1642,7 +1755,117 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
|
||||
).run()
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
return self.response(200, message="OK")
|
||||
return self.response(201, message="OK")
|
||||
|
||||
@expose("/columnar_metadata/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
|
||||
".columnar_metadata",
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
def columnar_metadata(self) -> Response:
|
||||
"""Upload a Columnar file and returns file metadata.
|
||||
---
|
||||
post:
|
||||
summary: Upload a Columnar file and returns file metadata
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ColumnarMetadataUploadFilePostSchema'
|
||||
responses:
|
||||
200:
|
||||
description: Columnar upload response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
result:
|
||||
$ref: '#/components/schemas/UploadFileMetadata'
|
||||
400:
|
||||
$ref: '#/components/responses/400'
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
404:
|
||||
$ref: '#/components/responses/404'
|
||||
500:
|
||||
$ref: '#/components/responses/500'
|
||||
"""
|
||||
try:
|
||||
request_form = request.form.to_dict()
|
||||
request_form["file"] = request.files.get("file")
|
||||
parameters = ColumnarMetadataUploadFilePostSchema().load(request_form)
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
metadata = ColumnarReader().file_metadata(parameters["file"])
|
||||
return self.response(200, result=UploadFileMetadata().dump(metadata))
|
||||
|
||||
@expose("/<int:pk>/columnar_upload/", methods=("POST",))
|
||||
@protect()
|
||||
@statsd_metrics
|
||||
@event_logger.log_this_with_context(
|
||||
action=lambda self,
|
||||
*args,
|
||||
**kwargs: f"{self.__class__.__name__}.columnar_upload",
|
||||
log_to_statsd=False,
|
||||
)
|
||||
@requires_form_data
|
||||
def columnar_upload(self, pk: int) -> Response:
|
||||
"""Upload a Columnar file into a database.
|
||||
---
|
||||
post:
|
||||
summary: Upload a Columnar file to a database table
|
||||
parameters:
|
||||
- in: path
|
||||
schema:
|
||||
type: integer
|
||||
name: pk
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ColumnarUploadPostSchema'
|
||||
responses:
|
||||
201:
|
||||
description: Columnar upload response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
400:
|
||||
$ref: '#/components/responses/400'
|
||||
401:
|
||||
$ref: '#/components/responses/401'
|
||||
404:
|
||||
$ref: '#/components/responses/404'
|
||||
422:
|
||||
$ref: '#/components/responses/422'
|
||||
500:
|
||||
$ref: '#/components/responses/500'
|
||||
"""
|
||||
try:
|
||||
request_form = request.form.to_dict()
|
||||
request_form["file"] = request.files.get("file")
|
||||
parameters = ColumnarUploadPostSchema().load(request_form)
|
||||
UploadCommand(
|
||||
pk,
|
||||
parameters["table_name"],
|
||||
parameters["file"],
|
||||
parameters.get("schema"),
|
||||
ColumnarReader(parameters),
|
||||
).run()
|
||||
except ValidationError as error:
|
||||
return self.response_400(message=error.messages)
|
||||
return self.response(201, message="OK")
|
||||
|
||||
@expose("/<int:pk>/function_names/", methods=("GET",))
|
||||
@protect()
|
||||
|
||||
Reference in New Issue
Block a user