QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Repeated boilerplate code between upload to database forms (#16756)

Browse Source 
* abstract boilerplate code into class and rename csv to file

* add db migration

* fix some stuff

* more renaming of csv to file

* rename in translations

* update down revision

* update down revision

* bump chart version

* switch to alter column name approach in db migration

* fix db migration for MySQL

* db migration conflict
This commit is contained in:
Shiva Raisinghani
2021-10-25 03:53:06 -07:00
committed by GitHub
parent 4f1d202430
commit ef3afbde82
53 changed files with 205 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
superset/views/database/forms.py
View File

@@ -42,53 +42,55 @@ from superset.models.core import Database
config = app.config
class CsvToDatabaseForm(DynamicForm):
class UploadToDatabaseForm(DynamicForm):
# pylint: disable=E0211
def csv_allowed_dbs() -> List[Database]: # type: ignore
csv_enabled_dbs = (
db.session.query(Database).filter_by(allow_csv_upload=True).all()
def file_allowed_dbs() -> List[Database]: # type: ignore
file_enabled_dbs = (
db.session.query(Database).filter_by(allow_file_upload=True).all()
)
return [
csv_enabled_db
for csv_enabled_db in csv_enabled_dbs
if CsvToDatabaseForm.at_least_one_schema_is_allowed(csv_enabled_db)
file_enabled_db
for file_enabled_db in file_enabled_dbs
if UploadToDatabaseForm.at_least_one_schema_is_allowed(file_enabled_db)
]
@staticmethod
def at_least_one_schema_is_allowed(database: Database) -> bool:
"""
If the user has access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
1. if schemas_allowed_for_file_upload is empty
a) if database does not support schema
user is able to upload csv without specifying schema name
b) if database supports schema
user is able to upload csv to any schema
2. if schemas_allowed_for_csv_upload is not empty
2. if schemas_allowed_for_file_upload is not empty
a) if database does not support schema
This situation is impossible and upload will fail
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
user is able to upload to schema in schemas_allowed_for_file_upload
elif the user does not access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
1. if schemas_allowed_for_file_upload is empty
a) if database does not support schema
user is unable to upload csv
b) if database supports schema
user is unable to upload csv
2. if schemas_allowed_for_csv_upload is not empty
2. if schemas_allowed_for_file_upload is not empty
a) if database does not support schema
This situation is impossible and user is unable to upload csv
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
user is able to upload to schema in schemas_allowed_for_file_upload
"""
if security_manager.can_access_database(database):
return True
schemas = database.get_schema_access_for_csv_upload()
schemas = database.get_schema_access_for_file_upload()
if schemas and security_manager.get_schemas_accessible_by_user(
database, schemas, False
):
return True
return False
class CsvToDatabaseForm(UploadToDatabaseForm):
name = StringField(
_("Table Name"),
description=_("Name of table to be created from csv data."),
@@ -116,7 +118,7 @@ class CsvToDatabaseForm(DynamicForm):
)
con = QuerySelectField(
_("Database"),
query_factory=csv_allowed_dbs,
query_factory=UploadToDatabaseForm.file_allowed_dbs,
get_pk=lambda a: a.id,
get_label=lambda a: a.database_name,
)
@@ -239,54 +241,7 @@ class CsvToDatabaseForm(DynamicForm):
)
class ExcelToDatabaseForm(DynamicForm):
# pylint: disable=E0211
def excel_allowed_dbs() -> List[Database]: # type: ignore
# TODO: change allow_csv_upload to allow_file_upload
excel_enabled_dbs = (
db.session.query(Database).filter_by(allow_csv_upload=True).all()
)
return [
excel_enabled_db
for excel_enabled_db in excel_enabled_dbs
if ExcelToDatabaseForm.at_least_one_schema_is_allowed(excel_enabled_db)
]
@staticmethod
def at_least_one_schema_is_allowed(database: Database) -> bool:
"""
If the user has access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
a) if database does not support schema
user is able to upload excel without specifying schema name
b) if database supports schema
user is able to upload excel to any schema
2. if schemas_allowed_for_csv_upload is not empty
a) if database does not support schema
This situation is impossible and upload will fail
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
elif the user does not access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
a) if database does not support schema
user is unable to upload excel
b) if database supports schema
user is unable to upload excel
2. if schemas_allowed_for_csv_upload is not empty
a) if database does not support schema
This situation is impossible and user is unable to upload excel
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
"""
if security_manager.can_access_database(database):
return True
schemas = database.get_schema_access_for_csv_upload()
if schemas and security_manager.schemas_accessible_by_user(
database, schemas, False
):
return True
return False
class ExcelToDatabaseForm(UploadToDatabaseForm):
name = StringField(
_("Table Name"),
description=_("Name of table to be created from excel data."),
@@ -322,7 +277,7 @@ class ExcelToDatabaseForm(DynamicForm):
con = QuerySelectField(
_("Database"),
query_factory=excel_allowed_dbs,
query_factory=UploadToDatabaseForm.file_allowed_dbs,
get_pk=lambda a: a.id,
get_label=lambda a: a.database_name,
)
@@ -419,56 +374,7 @@ class ExcelToDatabaseForm(DynamicForm):
)
class ColumnarToDatabaseForm(DynamicForm):
# pylint: disable=E0211
def columnar_allowed_dbs() -> List[Database]: # type: ignore
# TODO: change allow_csv_upload to allow_file_upload
columnar_enabled_dbs = (
db.session.query(Database).filter_by(allow_csv_upload=True).all()
)
return [
columnar_enabled_db
for columnar_enabled_db in columnar_enabled_dbs
if ColumnarToDatabaseForm.at_least_one_schema_is_allowed(
columnar_enabled_db
)
]
@staticmethod
def at_least_one_schema_is_allowed(database: Database) -> bool:
"""
If the user has access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
a) if database does not support schema
user is able to upload columnar without specifying schema name
b) if database supports schema
user is able to upload columnar to any schema
2. if schemas_allowed_for_csv_upload is not empty
a) if database does not support schema
This situation is impossible and upload will fail
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
elif the user does not access to the database or all datasource
1. if schemas_allowed_for_csv_upload is empty
a) if database does not support schema
user is unable to upload columnar
b) if database supports schema
user is unable to upload columnar
2. if schemas_allowed_for_csv_upload is not empty
a) if database does not support schema
This situation is impossible and user is unable to upload columnar
b) if database supports schema
user is able to upload to schema in schemas_allowed_for_csv_upload
"""
if security_manager.can_access_database(database):
return True
schemas = database.get_schema_access_for_csv_upload()
if schemas and security_manager.schemas_accessible_by_user(
database, schemas, False
):
return True
return False
class ColumnarToDatabaseForm(UploadToDatabaseForm):
name = StringField(
_("Table Name"),
description=_("Name of table to be created from columnar data."),
@@ -499,7 +405,7 @@ class ColumnarToDatabaseForm(DynamicForm):
con = QuerySelectField(
_("Database"),
query_factory=columnar_allowed_dbs,
query_factory=UploadToDatabaseForm.file_allowed_dbs,
get_pk=lambda a: a.id,
get_label=lambda a: a.database_name,
)

12
superset/views/database/mixins.py
View File

@@ -48,7 +48,7 @@ class DatabaseMixin:
"allow_run_async",
"allow_dml",
"modified",
"allow_csv_upload",
"allow_file_upload",
"expose_in_sqllab",
]
add_columns = [
@@ -57,7 +57,7 @@ class DatabaseMixin:
"cache_timeout",
"expose_in_sqllab",
"allow_run_async",
"allow_csv_upload",
"allow_file_upload",
"allow_ctas",
"allow_cvas",
"allow_dml",
@@ -136,9 +136,9 @@ class DatabaseMixin:
'"table_cache_timeout": 600}**. '
"If unset, cache will not be enabled for the functionality. "
"A timeout of 0 indicates that the cache never expires.<br/>"
"3. The ``schemas_allowed_for_csv_upload`` is a comma separated list "
"3. The ``schemas_allowed_for_file_upload`` is a comma separated list "
"of schemas that CSVs are allowed to upload to. "
'Specify it as **"schemas_allowed_for_csv_upload": '
'Specify it as **"schemas_allowed_for_file_upload": '
'["public", "csv_upload"]**. '
"If database flavor does not support schema or any schema is allowed "
"to be accessed, just leave the list empty<br/>"
@@ -177,7 +177,7 @@ class DatabaseMixin:
"A timeout of 0 indicates that the cache never expires. "
"Note this defaults to the global timeout if undefined."
),
"allow_csv_upload": _(
"allow_file_upload": _(
"If selected, please set the schemas allowed for csv upload in Extra."
),
}
@@ -198,7 +198,7 @@ class DatabaseMixin:
"server_cert": _("Root certificate"),
"allow_run_async": _("Async Execution"),
"impersonate_user": _("Impersonate the logged on user"),
"allow_csv_upload": _("Allow Csv Upload"),
"allow_file_upload": _("Allow Csv Upload"),
"modified": _("Modified"),
"allow_multi_schema_metadata_fetch": _("Allow Multi Schema Metadata Fetch"),
"backend": _("Backend"),

6
superset/views/database/validators.py
View File

@@ -48,10 +48,10 @@ def sqlalchemy_uri_validator(
) from ex
def schema_allows_csv_upload(database: Database, schema: Optional[str]) -> bool:
if not database.allow_csv_upload:
def schema_allows_file_upload(database: Database, schema: Optional[str]) -> bool:
if not database.allow_file_upload:
return False
schemas = database.get_schema_access_for_csv_upload()
schemas = database.get_schema_access_for_file_upload()
if schemas:
return schema in schemas
return security_manager.can_access_database(database)

8
superset/views/database/views.py
View File

@@ -43,7 +43,7 @@ from superset.views.base import DeleteMixin, SupersetModelView, YamlExportMixin
from .forms import ColumnarToDatabaseForm, CsvToDatabaseForm, ExcelToDatabaseForm
from .mixins import DatabaseMixin
from .validators import schema_allows_csv_upload, sqlalchemy_uri_validator
from .validators import schema_allows_file_upload, sqlalchemy_uri_validator
if TYPE_CHECKING:
from werkzeug.datastructures import FileStorage
@@ -132,7 +132,7 @@ class CsvToDatabaseView(SimpleFormView):
database = form.con.data
csv_table = Table(table=form.name.data, schema=form.schema.data)
if not schema_allows_csv_upload(database, csv_table.schema):
if not schema_allows_file_upload(database, csv_table.schema):
message = _(
'Database "%(database_name)s" schema "%(schema_name)s" '
"is not allowed for csv uploads. Please contact your Superset Admin.",
@@ -279,7 +279,7 @@ class ExcelToDatabaseView(SimpleFormView):
database = form.con.data
excel_table = Table(table=form.name.data, schema=form.schema.data)
if not schema_allows_csv_upload(database, excel_table.schema):
if not schema_allows_file_upload(database, excel_table.schema):
message = _(
'Database "%(database_name)s" schema "%(schema_name)s" '
"is not allowed for excel uploads. Please contact your Superset Admin.",
@@ -448,7 +448,7 @@ class ColumnarToDatabaseView(SimpleFormView):
"columns": form.usecols.data if form.usecols.data else None,
}
if not schema_allows_csv_upload(database, columnar_table.schema):
if not schema_allows_file_upload(database, columnar_table.schema):
message = _(
'Database "%(database_name)s" schema "%(schema_name)s" '
"is not allowed for columnar uploads. "