Add docstrings and typing to db_engine_specs and sql_parse (#8058)

* Add typing to db_engine_specs

* Add more type annotations and docstrings

* Add docstrings and typing to sql_parse and db_engine_specs

* Refine select_star

* Fix execute and add more docstrings

* Revert kwargs change from execute

* Remove redundant or

* Align view and table getter schema types

* Fix return type of latest_partition

* Remove some typing from presto

* Improve docstring for __extract_from_token

superset/sql_parse.py

@@ -16,7 +16,7 @@
 # under the License.
 # pylint: disable=C,R,W
 import logging
-from typing import Optional
+from typing import List, Optional, Set
 
 import sqlparse
 from sqlparse.sql import Identifier, IdentifierList, remove_quotes, Token, TokenList
@@ -31,10 +31,10 @@ CTE_PREFIX = "CTE__"
 
 class ParsedQuery(object):
     def __init__(self, sql_statement):
-        self.sql = sql_statement
-        self._table_names = set()
-        self._alias_names = set()
-        self._limit = None
+        self.sql: str = sql_statement
+        self._table_names: Set[str] = set()
+        self._alias_names: Set[str] = set()
+        self._limit: Optional[int] = None
 
         logging.info("Parsing with sqlparse statement {}".format(self.sql))
         self._parsed = sqlparse.parse(self.stripped())
@@ -44,27 +44,27 @@ class ParsedQuery(object):
         self._table_names = self._table_names - self._alias_names
 
     @property
-    def tables(self):
+    def tables(self) -> Set[str]:
         return self._table_names
 
     @property
-    def limit(self):
+    def limit(self) -> Optional[int]:
         return self._limit
 
-    def is_select(self):
+    def is_select(self) -> bool:
         return self._parsed[0].get_type() == "SELECT"
 
-    def is_explain(self):
+    def is_explain(self) -> bool:
         return self.stripped().upper().startswith("EXPLAIN")
 
-    def is_readonly(self):
+    def is_readonly(self) -> bool:
         """Pessimistic readonly, 100% sure statement won't mutate anything"""
         return self.is_select() or self.is_explain()
 
-    def stripped(self):
+    def stripped(self) -> str:
         return self.sql.strip(" \t\n;")
 
-    def get_statements(self):
+    def get_statements(self) -> List[str]:
         """Returns a list of SQL statements as strings, stripped"""
         statements = []
         for statement in self._parsed:
@@ -105,36 +105,39 @@ class ParsedQuery(object):
         return None
 
     @staticmethod
-    def __is_identifier(token: Token):
+    def __is_identifier(token: Token) -> bool:
         return isinstance(token, (IdentifierList, Identifier))
 
-    def __process_tokenlist(self, tlist: TokenList):
+    def __process_tokenlist(self, token_list: TokenList):
+        """
+        Add table names to table set
+
+        :param token_list: TokenList to be processed
+        """
         # exclude subselects
-        if "(" not in str(tlist):
-            table_name = self.__get_full_name(tlist)
+        if "(" not in str(token_list):
+            table_name = self.__get_full_name(token_list)
             if table_name and not table_name.startswith(CTE_PREFIX):
                 self._table_names.add(table_name)
             return
 
         # store aliases
-        if tlist.has_alias():
-            self._alias_names.add(tlist.get_alias())
+        if token_list.has_alias():
+            self._alias_names.add(token_list.get_alias())
 
         # some aliases are not parsed properly
-        if tlist.tokens[0].ttype == Name:
-            self._alias_names.add(tlist.tokens[0].value)
-        self.__extract_from_token(tlist)
+        if token_list.tokens[0].ttype == Name:
+            self._alias_names.add(token_list.tokens[0].value)
+        self.__extract_from_token(token_list)
 
-    def as_create_table(self, table_name, overwrite=False):
+    def as_create_table(self, table_name: str, overwrite: bool = False) -> str:
         """Reformats the query into the create table as query.
 
         Works only for the single select SQL statements, in all other cases
         the sql query is not modified.
-        :param superset_query: string, sql query that will be executed
-        :param table_name: string, will contain the results of the
-            query execution
-        :param overwrite, boolean, table table_name will be dropped if true
-        :return: string, create table as query
+        :param table_name: Table that will contain the results of the query execution
+        :param overwrite: table_name will be dropped if true
+        :return: Create table as query
         """
         exec_sql = ""
         sql = self.stripped()
@@ -143,7 +146,12 @@ class ParsedQuery(object):
         exec_sql += f"CREATE TABLE {table_name} AS \n{sql}"
         return exec_sql
 
-    def __extract_from_token(self, token, depth=0):
+    def __extract_from_token(self, token: Token):
+        """
+        Populate self._table_names from token
+
+        :param token: instance of Token or child class, e.g. TokenList, to be processed
+        """
         if not hasattr(token, "tokens"):
             return
 
@@ -151,7 +159,7 @@ class ParsedQuery(object):
 
         for item in token.tokens:
             if item.is_group and not self.__is_identifier(item):
-                self.__extract_from_token(item, depth=depth + 1)
+                self.__extract_from_token(item)
 
             if item.ttype in Keyword and (
                 item.normalized in PRECEDES_TABLE_NAME
@@ -174,9 +182,15 @@ class ParsedQuery(object):
             elif isinstance(item, IdentifierList):
                 for token in item.tokens:
                     if not self.__is_identifier(token):
-                        self.__extract_from_token(item, depth=depth + 1)
+                        self.__extract_from_token(item)
 
-    def _extract_limit_from_query(self, statement):
+    def _extract_limit_from_query(self, statement: TokenList) -> Optional[int]:
+        """
+        Extract limit clause from SQL statement.
+
+        :param statement: SQL statement
+        :return: Limit extracted from query, None if no limit present in statement
+        """
         idx, _ = statement.token_next_by(m=(Keyword, "LIMIT"))
         if idx is not None:
             _, token = statement.token_next(idx=idx)
@@ -188,10 +202,16 @@ class ParsedQuery(object):
                     _, token = token.token_next(idx=idx)
                 if token and token.ttype == sqlparse.tokens.Literal.Number.Integer:
                     return int(token.value)
+        return None
 
-    def get_query_with_new_limit(self, new_limit):
-        """returns the query with the specified limit"""
-        """does not change the underlying query"""
+    def get_query_with_new_limit(self, new_limit: int) -> str:
+        """
+        returns the query with the specified limit.
+        Does not change the underlying query
+
+        :param new_limit: Limit to be incorporated into returned query
+        :return: The original query with new limit
+        """
         if not self._limit:
             return f"{self.stripped()}\nLIMIT {new_limit}"
         limit_pos = None