QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Upgrade pylint to 2.5.3 and fix most new rules (#10101)

Browse Source 
* Bump pylint version to 2.5.3

* Add a global disable for the most common new pylint error

* Fix a bunch of files containing very few errors

* More pylint tweakage, low-hanging fruit

* More easy stuff...

* Fix more erroring files

* Fix the last couple of errors, clean pylint!

* Black

* Fix mypy issue in connectors/druid/models.py
This commit is contained in:
Will Barrett
2020-06-18 14:03:42 -07:00
committed by GitHub
parent 02fee35314
commit 8e23d4f369
40 changed files with 166 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
superset/db_engine_specs/base.py
View File

@@ -332,7 +332,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
set to is_dttm=True. Note that this only gets called when new
columns are detected/created"""
# TODO: Fix circular import caused by importing TableColumn
pass
@classmethod
def epoch_to_dttm(cls) -> str:
@@ -401,9 +400,11 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
.limit(limit)
)
return database.compile_sqla_query(qry)
elif LimitMethod.FORCE_LIMIT:
if LimitMethod.FORCE_LIMIT:
parsed_query = sql_parse.ParsedQuery(sql)
sql = parsed_query.set_or_update_query_limit(limit)
return sql
@classmethod
@@ -465,7 +466,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
Create table from contents of a csv. Note: this method does not create
metadata for the table.
"""
df = cls.csv_to_df(filepath_or_buffer=filename, **csv_to_df_kwargs,)
df = cls.csv_to_df(filepath_or_buffer=filename, **csv_to_df_kwargs)
engine = cls.get_engine(database)
if table.schema:
# only add schema when it is preset and non empty
@@ -529,7 +530,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
for handling the cursor and updating progress information in the
query object"""
# TODO: Fix circular import error caused by importing sql_lab.Query
pass
@classmethod
def extract_error_message(cls, ex: Exception) -> str:
@@ -573,14 +573,12 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
Some database drivers like presto accept '{catalog}/{schema}' in
the database component of the URL, that can be handled here.
"""
pass
@classmethod
def patch(cls) -> None:
"""
TODO: Improve docstring and refactor implementation in Hive
"""
pass
@classmethod
def get_schema_names(cls, inspector: Inspector) -> List[str]:

2
superset/db_engine_specs/drill.py
View File

@@ -56,7 +56,7 @@ class DrillEngineSpec(BaseEngineSpec):
tt = target_type.upper()
if tt == "DATE":
return f"TO_DATE('{dttm.date().isoformat()}', 'yyyy-MM-dd')"
elif tt == "TIMESTAMP":
if tt == "TIMESTAMP":
return f"""TO_TIMESTAMP('{dttm.isoformat(sep=" ", timespec="seconds")}', 'yyyy-MM-dd HH:mm:ss')""" # pylint: disable=line-too-long
return None

19
superset/db_engine_specs/hive.py
View File

@@ -200,7 +200,7 @@ class HiveEngineSpec(PrestoEngineSpec):
tt = target_type.upper()
if tt == "DATE":
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
elif tt == "TIMESTAMP":
if tt == "TIMESTAMP":
return f"""CAST('{dttm.isoformat(sep=" ", timespec="microseconds")}' AS TIMESTAMP)""" # pylint: disable=line-too-long
return None
@@ -284,7 +284,9 @@ class HiveEngineSpec(PrestoEngineSpec):
if log:
log_lines = log.splitlines()
progress = cls.progress(log_lines)
logger.info(f"Query {query_id}: Progress total: {progress}")
logger.info(
"Query %s: Progress total: %s", str(query_id), str(progress)
)
needs_commit = False
if progress > query.progress:
query.progress = progress
@@ -294,21 +296,25 @@ class HiveEngineSpec(PrestoEngineSpec):
if tracking_url:
job_id = tracking_url.split("/")[-2]
logger.info(
f"Query {query_id}: Found the tracking url: {tracking_url}"
"Query %s: Found the tracking url: %s",
str(query_id),
tracking_url,
)
tracking_url = tracking_url_trans(tracking_url)
logger.info(
f"Query {query_id}: Transformation applied: {tracking_url}"
"Query %s: Transformation applied: %s",
str(query_id),
tracking_url,
)
query.tracking_url = tracking_url
logger.info(f"Query {query_id}: Job id: {job_id}")
logger.info("Query %s: Job id: %s", str(query_id), str(job_id))
needs_commit = True
if job_id and len(log_lines) > last_log_line:
# Wait for job id before logging things out
# this allows for prefixing all log lines and becoming
# searchable in something like Kibana
for l in log_lines[last_log_line:]:
logger.info(f"Query {query_id}: [{job_id}] {l}")
logger.info("Query %s: [%s] %s", str(query_id), str(job_id), l)
last_log_line = len(log_lines)
if needs_commit:
session.commit()
@@ -414,7 +420,6 @@ class HiveEngineSpec(PrestoEngineSpec):
"""
# Do nothing in the URL object since instead this should modify
# the configuraiton dictionary. See get_configuration_for_impersonation
pass
@classmethod
def get_configuration_for_impersonation(

2
superset/db_engine_specs/impala.py
View File

@@ -47,7 +47,7 @@ class ImpalaEngineSpec(BaseEngineSpec):
tt = target_type.upper()
if tt == "DATE":
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
elif tt == "TIMESTAMP":
if tt == "TIMESTAMP":
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS TIMESTAMP)"""
return None

4
superset/db_engine_specs/pinot.py
View File

@@ -84,7 +84,9 @@ class PinotEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
if not granularity:
raise NotImplementedError("No pinot grain spec for " + str(time_grain))
else:
return TimestampExpression(f"{{col}}", col)
return TimestampExpression(
f"{{col}}", col # pylint: disable=f-string-without-interpolation
)
# In pinot the output is a string since there is no timestamp column like pg
time_expr = f"DATETIMECONVERT({{col}}, '{tf}', '{tf}', '{granularity}')"
return TimestampExpression(time_expr, col)

24
superset/db_engine_specs/presto.py
View File

@@ -79,7 +79,8 @@ def get_children(column: Dict[str, str]) -> List[Dict[str, str]]:
children_type = group["children"]
if type_ == "ARRAY":
return [{"name": column["name"], "type": children_type}]
elif type_ == "ROW":
if type_ == "ROW":
nameless_columns = 0
columns = []
for child in utils.split(children_type, ","):
@@ -93,8 +94,8 @@ def get_children(column: Dict[str, str]) -> List[Dict[str, str]]:
nameless_columns += 1
columns.append({"name": f"{column['name']}.{name.lower()}", "type": type_})
return columns
else:
raise Exception(f"Unknown type {type_}!")
raise Exception(f"Unknown type {type_}!")
class PrestoEngineSpec(BaseEngineSpec):
@@ -278,7 +279,7 @@ class PrestoEngineSpec(BaseEngineSpec):
if not (inner_type.endswith("array") or inner_type.endswith("row")):
stack.pop()
# We have an array of row objects (i.e. array(row(...)))
elif inner_type == "array" or inner_type == "row":
elif inner_type in ("array", "row"):
# Push a dummy object to represent the structural data type
stack.append(("", inner_type))
# We have an array of a basic data types(i.e. array(varchar)).
@@ -339,8 +340,9 @@ class PrestoEngineSpec(BaseEngineSpec):
)
result[structural_column_index]["default"] = None
continue
else: # otherwise column is a basic data type
column_type = presto_type_map[column.Type]()
# otherwise column is a basic data type
column_type = presto_type_map[column.Type]()
except KeyError:
logger.info(
"Did not recognize type {} of column {}".format( # pylint: disable=logging-format-interpolation
@@ -727,7 +729,7 @@ class PrestoEngineSpec(BaseEngineSpec):
def handle_cursor(cls, cursor: Any, query: Query, session: Session) -> None:
"""Updates progress information"""
query_id = query.id
logger.info(f"Query {query_id}: Polling the cursor for progress")
logger.info("Query %i: Polling the cursor for progress", query_id)
polled = cursor.poll()
# poll returns dict -- JSON status information or ``None``
# if the query is done
@@ -761,7 +763,7 @@ class PrestoEngineSpec(BaseEngineSpec):
query.progress = progress
session.commit()
time.sleep(1)
logger.info(f"Query {query_id}: Polling the cursor for progress")
logger.info("Query %i: Polling the cursor for progress", query_id)
polled = cursor.poll()
@classmethod
@@ -903,12 +905,14 @@ class PrestoEngineSpec(BaseEngineSpec):
raise SupersetTemplateException(
"The table should have one partitioned field"
)
elif not show_first and len(indexes[0]["column_names"]) > 1:
if not show_first and len(indexes[0]["column_names"]) > 1:
raise SupersetTemplateException(
"The table should have a single partitioned field "
"to use this function. You may want to use "
"`presto.latest_sub_partition`"
)
column_names = indexes[0]["column_names"]
part_fields = [(column_name, True) for column_name in column_names]
sql = cls._partition_query(table_name, database, 1, part_fields)
@@ -947,7 +951,7 @@ class PrestoEngineSpec(BaseEngineSpec):
indexes = database.get_indexes(table_name, schema)
part_fields = indexes[0]["column_names"]
for k in kwargs.keys(): # pylint: disable=consider-iterating-dictionary
if k not in k in part_fields:
if k not in k in part_fields: # pylint: disable=comparison-with-itself
msg = "Field [{k}] is not part of the portioning key"
raise SupersetTemplateException(msg)
if len(kwargs.keys()) != len(part_fields) - 1:

5
superset/db_engine_specs/sqlite.py
View File

@@ -64,15 +64,14 @@ class SqliteEngineSpec(BaseEngineSpec):
cache=database.table_cache_enabled,
cache_timeout=database.table_cache_timeout,
)
elif datasource_type == "view":
if datasource_type == "view":
return database.get_all_view_names_in_schema(
schema=schema,
force=True,
cache=database.table_cache_enabled,
cache_timeout=database.table_cache_timeout,
)
else:
raise Exception(f"Unsupported datasource_type: {datasource_type}")
raise Exception(f"Unsupported datasource_type: {datasource_type}")
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: