diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80202bebfab..17de147a6b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: rev: v0.910 hooks: - id: mypy - additional_dependencies: [types-all] + additional_dependencies: [types-all, types-redis] - repo: https://github.com/peterdemin/pip-compile-multi rev: v2.4.1 hooks: diff --git a/requirements/base.in b/requirements/base.in index f2b8a7c4403..30482c5f826 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -19,3 +19,7 @@ pyrsistent>=0.16.1,<0.17 zipp==3.4.1 sasl==0.3.1 +packaging==21.0 +wrapt==1.12.1 +certifi==2021.5.30 +charset-normalizer==2.0.4 diff --git a/requirements/base.txt b/requirements/base.txt index c8b9c69b012..cbab2a8bb62 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:04efc15075d69b1a2b5fa6c76b84c77a2f5c04e3 +# SHA1:fe363b0ea02d7589c2ba5a1cf936247a966a6d5e # # This file is autogenerated by pip-compile-multi # To update, run: @@ -35,10 +35,18 @@ cachelib==0.1.1 # via apache-superset celery==4.4.7 # via apache-superset +certifi==2021.5.30 + # via + # -r requirements/base.in + # requests cffi==1.14.6 # via cryptography chardet==4.0.0 # via aiohttp +charset-normalizer==2.0.4 + # via + # -r requirements/base.in + # requests click==7.1.2 # via # apache-superset @@ -118,6 +126,7 @@ humanize==3.11.0 idna==3.2 # via # email-validator + # requests # yarl isodate==0.6.0 # via apache-superset @@ -166,6 +175,7 @@ numpy==1.21.1 # pyarrow packaging==21.0 # via + # -r requirements/base.in # bleach # deprecation pandas==1.2.5 @@ -226,6 +236,8 @@ pyyaml==5.4.1 # apispec redis==3.5.3 # via apache-superset +requests==2.26.0 + # via apache-superset sasl==0.3.1 # via -r requirements/base.in selenium==3.141.0 @@ -270,7 +282,9 @@ typing-extensions==3.10.0.0 # aiohttp # apache-superset urllib3==1.26.6 - # via selenium + # via + # requests + # selenium vine==1.3.0 # via # amqp @@ -281,6 +295,8 @@ werkzeug==1.0.1 # via # flask # flask-jwt-extended +wrapt==1.12.1 + # via -r requirements/base.in wtforms==2.3.3 # via # flask-wtf diff --git a/requirements/development.txt b/requirements/development.txt index 3a62a283865..c4c9aec2a65 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -16,10 +16,6 @@ botocore==1.21.19 # s3transfer cached-property==1.5.2 # via tableschema -certifi==2021.5.30 - # via requests -charset-normalizer==2.0.4 - # via requests et-xmlfile==1.1.0 # via openpyxl flask-cors==3.0.10 @@ -54,11 +50,6 @@ pyhive[hive]==0.6.4 # via -r requirements/development.in pyinstrument==4.0.2 # via -r requirements/development.in -requests==2.26.0 - # via - # pydruid - # tableschema - # tabulator rfc3986==1.5.0 # via tableschema s3transfer==0.5.0 diff --git a/requirements/integration.in b/requirements/integration.in index 1bc94c7fb71..305d55a8ab0 100644 --- a/requirements/integration.in +++ b/requirements/integration.in @@ -20,3 +20,4 @@ tox py>=1.10.0 click==7.1.2 packaging==21.0 +pyparsing==2.4.7 diff --git a/requirements/integration.txt b/requirements/integration.txt index b7da99e3da3..0cceda20be0 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -1,4 +1,4 @@ -# SHA1:17ab2346746deadfc557e1df96014e77c8337f4b +# SHA1:32bae3a7c758a411c20c86ff4d5bff825be46314 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -45,7 +45,9 @@ py==1.10.0 # -r requirements/integration.in # tox pyparsing==2.4.7 - # via packaging + # via + # -r requirements/integration.in + # packaging pyyaml==5.4.1 # via pre-commit six==1.16.0 diff --git a/requirements/testing.txt b/requirements/testing.txt index 00492fb58dd..68c47b05c6f 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -92,8 +92,6 @@ wcwidth==0.2.5 # via prompt-toolkit websocket-client==1.2.0 # via docker -wrapt==1.12.1 - # via astroid # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/setup.py b/setup.py index 42eedd36deb..1dae60db497 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ setup( "pyyaml>=5.4", "PyJWT>=1.7.1, <2", "redis", + "requests==2.26.0", "selenium>=3.141.0", "simplejson>=3.15.0", "slackclient==2.5.0", # PINNED! slack changes file upload api in the future versions diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js b/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js index 14e0ce31462..a4d3fbaa815 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js @@ -67,7 +67,7 @@ describe('Dashboard save action', () => { // should load chart WORLD_HEALTH_CHARTS.forEach(waitForChartLoad); - // remove box_plot chart from dashboard + // remove treemap chart from dashboard cy.get('[aria-label="edit-alt"]').click({ timeout: 5000 }); cy.get('[data-test="dashboard-delete-component-button"]') .last() diff --git a/superset/common/query_object.py b/superset/common/query_object.py index 6ef00cdb835..e05cad7d608 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -25,8 +25,6 @@ from pandas import DataFrame from superset import app, db from superset.connectors.base.models import BaseDatasource from superset.connectors.connector_registry import ConnectorRegistry -from superset.exceptions import QueryObjectValidationError -from superset.typing import Metric, OrderBy from superset.exceptions import ( QueryClauseValidationException, QueryObjectValidationError, diff --git a/superset/config.py b/superset/config.py index 54545dfda89..7f504582c59 100644 --- a/superset/config.py +++ b/superset/config.py @@ -38,7 +38,6 @@ from dateutil import tz from flask import Blueprint from flask_appbuilder.security.manager import AUTH_DB from pandas.io.parsers import STR_NA_VALUES -from typing_extensions import Literal from werkzeug.local import LocalProxy from superset.jinja_context import BaseTemplateProcessor diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 85e41a4b679..6495f305891 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -1452,7 +1452,9 @@ class SqlaTable(Model, BaseDatasource): # pylint: disable=too-many-public-metho # string into a timestamp. if column_map[dimension].is_temporal and isinstance(value, str): dttm = dateutil.parser.parse(value) - value = text(self.db_engine_spec.convert_dttm("TIMESTAMP", dttm)) + value = text( + str(self.db_engine_spec.convert_dttm("TIMESTAMP", dttm)) + ) group.append(groupby_exprs[dimension] == value) groups.append(and_(*group)) diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 47cf874ccbc..2e1f2908c15 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -32,7 +32,7 @@ from typing import ( ) from flask_babel import gettext as __ -from pytz import _FixedOffset # type: ignore +from pytz import _FixedOffset from sqlalchemy.dialects.postgresql import ARRAY, DOUBLE_PRECISION, ENUM, JSON from sqlalchemy.dialects.postgresql.base import PGInspector from sqlalchemy.types import String, TypeEngine diff --git a/superset/utils/async_query_manager.py b/superset/utils/async_query_manager.py index 14f89a83a10..b197a853cd9 100644 --- a/superset/utils/async_query_manager.py +++ b/superset/utils/async_query_manager.py @@ -199,9 +199,5 @@ class AsyncQueryManager: logger.debug("********** logging event data to stream %s", scoped_stream_name) logger.debug(event_data) - self._redis.xadd( # type: ignore - scoped_stream_name, event_data, "*", self._stream_limit - ) - self._redis.xadd( # type: ignore - full_stream_name, event_data, "*", self._stream_limit_firehose - ) + self._redis.xadd(scoped_stream_name, event_data, "*", self._stream_limit) + self._redis.xadd(full_stream_name, event_data, "*", self._stream_limit_firehose) diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index f77b6061680..ac4ef4842ef 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -2118,5 +2118,3 @@ class TestChartApi(SupersetTestCase, ApiOwnersTestCaseMixin, InsertChartMixin): assert "':asdf'" in result["query"] assert "':xyz:qwerty'" in result["query"] assert "':qwerty:'" in result["query"] - - diff --git a/tests/unit_tests/sql_parse_tests.py b/tests/unit_tests/sql_parse_tests.py index 63dfeeef15a..191e0f82ada 100644 --- a/tests/unit_tests/sql_parse_tests.py +++ b/tests/unit_tests/sql_parse_tests.py @@ -15,11 +15,7 @@ # specific language governing permissions and limitations # under the License. -from superset.sql_parse import ParsedQuery - - import pytest -import sqlparse from superset.exceptions import QueryClauseValidationException from superset.sql_parse import ( @@ -30,944 +26,7 @@ from superset.sql_parse import ( ) -def extract_tables(query: str) -> Set[Table]: - """ - Helper function to extract tables referenced in a query. - """ - return ParsedQuery(query).tables - - -def test_table() -> None: - """ - Test the ``Table`` class and its string conversion. - - Special characters in the table, schema, or catalog name should be escaped correctly. - """ - assert str(Table("table_name")) == "table_name" - assert str(Table("table_name", "schema_name")) == "schema_name.table_name" - assert ( - str(Table("table_name", "schema_name", "catalog_name")) - == "catalog_name.schema_name.table_name" - ) - assert ( - str(Table("table.name", "schema/name", "catalog\nname")) - == "catalog%0Aname.schema%2Fname.table%2Ename" - ) - - -def test_extract_tables() -> None: - """ - Test that referenced tables are parsed correctly from the SQL. - """ - assert extract_tables("SELECT * FROM tbname") == {Table("tbname")} - assert extract_tables("SELECT * FROM tbname foo") == {Table("tbname")} - assert extract_tables("SELECT * FROM tbname AS foo") == {Table("tbname")} - - # underscore - assert extract_tables("SELECT * FROM tb_name") == {Table("tb_name")} - - # quotes - assert extract_tables('SELECT * FROM "tbname"') == {Table("tbname")} - - # unicode - assert extract_tables('SELECT * FROM "tb_name" WHERE city = "Lübeck"') == { - Table("tb_name") - } - - # columns - assert extract_tables("SELECT field1, field2 FROM tb_name") == {Table("tb_name")} - assert extract_tables("SELECT t1.f1, t2.f2 FROM t1, t2") == { - Table("t1"), - Table("t2"), - } - - # named table - assert extract_tables("SELECT a.date, a.field FROM left_table a LIMIT 10") == { - Table("left_table") - } - - # reverse select - assert extract_tables("FROM t1 SELECT field") == {Table("t1")} - - -def test_extract_tables_subselect() -> None: - """ - Test that tables inside subselects are parsed correctly. - """ - assert ( - extract_tables( - """ -SELECT sub.* -FROM ( - SELECT * - FROM s1.t1 - WHERE day_of_week = 'Friday' - ) sub, s2.t2 -WHERE sub.resolution = 'NONE' -""" - ) - == {Table("t1", "s1"), Table("t2", "s2")} - ) - - assert ( - extract_tables( - """ -SELECT sub.* -FROM ( - SELECT * - FROM s1.t1 - WHERE day_of_week = 'Friday' -) sub -WHERE sub.resolution = 'NONE' -""" - ) - == {Table("t1", "s1")} - ) - - assert ( - extract_tables( - """ -SELECT * FROM t1 -WHERE s11 > ANY ( - SELECT COUNT(*) /* no hint */ FROM t2 - WHERE NOT EXISTS ( - SELECT * FROM t3 - WHERE ROW(5*t2.s1,77)=( - SELECT 50,11*s1 FROM t4 - ) - ) -) -""" - ) - == {Table("t1"), Table("t2"), Table("t3"), Table("t4")} - ) - - -def test_extract_tables_select_in_expression() -> None: - """ - Test that parser works with ``SELECT``s used as expressions. - """ - assert extract_tables("SELECT f1, (SELECT count(1) FROM t2) FROM t1") == { - Table("t1"), - Table("t2"), - } - assert extract_tables("SELECT f1, (SELECT count(1) FROM t2) as f2 FROM t1") == { - Table("t1"), - Table("t2"), - } - - -def test_extract_tables_parenthesis() -> None: - """ - Test that parenthesis are parsed correctly. - """ - assert extract_tables("SELECT f1, (x + y) AS f2 FROM t1") == {Table("t1")} - - -def test_extract_tables_with_schema() -> None: - """ - Test that schemas are parsed correctly. - """ - assert extract_tables("SELECT * FROM schemaname.tbname") == { - Table("tbname", "schemaname") - } - assert extract_tables('SELECT * FROM "schemaname"."tbname"') == { - Table("tbname", "schemaname") - } - assert extract_tables('SELECT * FROM "schemaname"."tbname" foo') == { - Table("tbname", "schemaname") - } - assert extract_tables('SELECT * FROM "schemaname"."tbname" AS foo') == { - Table("tbname", "schemaname") - } - - -def test_extract_tables_union() -> None: - """ - Test that ``UNION`` queries work as expected. - """ - assert extract_tables("SELECT * FROM t1 UNION SELECT * FROM t2") == { - Table("t1"), - Table("t2"), - } - assert extract_tables("SELECT * FROM t1 UNION ALL SELECT * FROM t2") == { - Table("t1"), - Table("t2"), - } - assert extract_tables("SELECT * FROM t1 INTERSECT ALL SELECT * FROM t2") == { - Table("t1"), - Table("t2"), - } - - -def test_extract_tables_select_from_values() -> None: - """ - Test that selecting from values returns no tables. - """ - assert extract_tables("SELECT * FROM VALUES (13, 42)") == set() - - -def test_extract_tables_select_array() -> None: - """ - Test that queries selecting arrays work as expected. - """ - assert ( - extract_tables( - """ -SELECT ARRAY[1, 2, 3] AS my_array -FROM t1 LIMIT 10 -""" - ) - == {Table("t1")} - ) - - -def test_extract_tables_select_if() -> None: - """ - Test that queries with an ``IF`` work as expected. - """ - assert ( - extract_tables( - """ -SELECT IF(CARDINALITY(my_array) >= 3, my_array[3], NULL) -FROM t1 LIMIT 10 -""" - ) - == {Table("t1")} - ) - - -def test_extract_tables_with_catalog() -> None: - """ - Test that catalogs are parsed correctly. - """ - assert extract_tables("SELECT * FROM catalogname.schemaname.tbname") == { - Table("tbname", "schemaname", "catalogname") - } - - -def test_extract_tables_illdefined() -> None: - """ - Test that ill-defined tables return an empty set. - """ - assert extract_tables("SELECT * FROM schemaname.") == set() - assert extract_tables("SELECT * FROM catalogname.schemaname.") == set() - assert extract_tables("SELECT * FROM catalogname..") == set() - assert extract_tables("SELECT * FROM catalogname..tbname") == set() - - -@unittest.skip("Requires sqlparse>=3.1") -def test_extract_tables_show_tables_from() -> None: - """ - Test ``SHOW TABLES FROM``. - - This is currently broken in the pinned version of sqlparse, and fixed in - ``sqlparse>=3.1``. However, ``sqlparse==3.1`` breaks some sql formatting. - """ - assert extract_tables("SHOW TABLES FROM s1 like '%order%'") == set() - - -def test_extract_tables_show_columns_from() -> None: - """ - Test ``SHOW COLUMNS FROM``. - """ - assert extract_tables("SHOW COLUMNS FROM t1") == {Table("t1")} - - -def test_extract_tables_where_subquery() -> None: - """ - Test that tables in a ``WHERE`` subquery are parsed correctly. - """ - assert ( - extract_tables( - """ -SELECT name -FROM t1 -WHERE regionkey = (SELECT max(regionkey) FROM t2) -""" - ) - == {Table("t1"), Table("t2")} - ) - - assert ( - extract_tables( - """ -SELECT name -FROM t1 -WHERE regionkey IN (SELECT regionkey FROM t2) -""" - ) - == {Table("t1"), Table("t2")} - ) - - assert ( - extract_tables( - """ -SELECT name -FROM t1 -WHERE regionkey EXISTS (SELECT regionkey FROM t2) -""" - ) - == {Table("t1"), Table("t2")} - ) - - -def test_extract_tables_describe() -> None: - """ - Test ``DESCRIBE``. - """ - assert extract_tables("DESCRIBE t1") == {Table("t1")} - - -def test_extract_tables_show_partitions() -> None: - """ - Test ``SHOW PARTITIONS``. - """ - assert ( - extract_tables( - """ -SHOW PARTITIONS FROM orders -WHERE ds >= '2013-01-01' ORDER BY ds DESC -""" - ) - == {Table("orders")} - ) - - -def test_extract_tables_join() -> None: - """ - Test joins. - """ - assert extract_tables("SELECT t1.*, t2.* FROM t1 JOIN t2 ON t1.a = t2.a;") == { - Table("t1"), - Table("t2"), - } - - assert ( - extract_tables( - """ -SELECT a.date, b.name -FROM left_table a -JOIN ( - SELECT - CAST((b.year) as VARCHAR) date, - name - FROM right_table -) b -ON a.date = b.date -""" - ) - == {Table("left_table"), Table("right_table")} - ) - - assert ( - extract_tables( - """ -SELECT a.date, b.name -FROM left_table a -LEFT INNER JOIN ( - SELECT - CAST((b.year) as VARCHAR) date, - name - FROM right_table -) b -ON a.date = b.date -""" - ) - == {Table("left_table"), Table("right_table")} - ) - - assert ( - extract_tables( - """ -SELECT a.date, b.name -FROM left_table a -RIGHT OUTER JOIN ( - SELECT - CAST((b.year) as VARCHAR) date, - name - FROM right_table -) b -ON a.date = b.date -""" - ) - == {Table("left_table"), Table("right_table")} - ) - - assert ( - extract_tables( - """ -SELECT a.date, b.name -FROM left_table a -FULL OUTER JOIN ( - SELECT - CAST((b.year) as VARCHAR) date, - name - FROM right_table -) b -ON a.date = b.date -""" - ) - == {Table("left_table"), Table("right_table")} - ) - - -def test_extract_tables_semi_join() -> None: - """ - Test ``LEFT SEMI JOIN``. - """ - assert ( - extract_tables( - """ -SELECT a.date, b.name -FROM left_table a -LEFT SEMI JOIN ( - SELECT - CAST((b.year) as VARCHAR) date, - name - FROM right_table -) b -ON a.data = b.date -""" - ) - == {Table("left_table"), Table("right_table")} - ) - - -def test_extract_tables_combinations() -> None: - """ - Test a complex case with nested queries. - """ - assert ( - extract_tables( - """ -SELECT * FROM t1 -WHERE s11 > ANY ( - SELECT * FROM t1 UNION ALL SELECT * FROM ( - SELECT t6.*, t3.* FROM t6 JOIN t3 ON t6.a = t3.a - ) tmp_join - WHERE NOT EXISTS ( - SELECT * FROM t3 - WHERE ROW(5*t3.s1,77)=( - SELECT 50,11*s1 FROM t4 - ) - ) -) -""" - ) - == {Table("t1"), Table("t3"), Table("t4"), Table("t6")} - ) - - assert ( - extract_tables( - """ -SELECT * FROM ( - SELECT * FROM ( - SELECT * FROM ( - SELECT * FROM EmployeeS - ) AS S1 - ) AS S2 -) AS S3 -""" - ) - == {Table("EmployeeS")} - ) - - -def test_extract_tables_with() -> None: - """ - Test ``WITH``. - """ - assert ( - extract_tables( - """ -WITH - x AS (SELECT a FROM t1), - y AS (SELECT a AS b FROM t2), - z AS (SELECT b AS c FROM t3) -SELECT c FROM z -""" - ) - == {Table("t1"), Table("t2"), Table("t3")} - ) - - assert ( - extract_tables( - """ -WITH - x AS (SELECT a FROM t1), - y AS (SELECT a AS b FROM x), - z AS (SELECT b AS c FROM y) -SELECT c FROM z -""" - ) - == {Table("t1")} - ) - - -def test_extract_tables_reusing_aliases() -> None: - """ - Test that the parser follows aliases. - """ - assert ( - extract_tables( - """ -with q1 as ( select key from q2 where key = '5'), -q2 as ( select key from src where key = '5') -select * from (select key from q1) a -""" - ) - == {Table("src")} - ) - - -def test_extract_tables_multistatement() -> None: - """ - Test that the parser works with multiple statements. - """ - assert extract_tables("SELECT * FROM t1; SELECT * FROM t2") == { - Table("t1"), - Table("t2"), - } - assert extract_tables("SELECT * FROM t1; SELECT * FROM t2;") == { - Table("t1"), - Table("t2"), - } - - -def test_extract_tables_keyword() -> None: - """ - Test that table names that are keywords work as expected. - - If the table name is a ``sqlparse`` reserved keyword (eg, "table_name") the parser - needs extra logic to identify it. - """ - assert extract_tables("SELECT * FROM table_name") == {Table("table_name")} - assert extract_tables("SELECT * FROM table_name AS foo") == {Table("table_name")} - - # these 3 are considered keywords - assert extract_tables("SELECT * FROM catalog_name.schema_name.table_name") == { - Table("table_name", "schema_name", "catalog_name") - } - - -def test_extract_tables_complex() -> None: - """ - Test a few complex queries. - """ - assert ( - extract_tables( - """ -SELECT sum(m_examples) AS "sum__m_example" -FROM ( - SELECT - COUNT(DISTINCT id_userid) AS m_examples, - some_more_info - FROM my_b_table b - JOIN my_t_table t ON b.ds=t.ds - JOIN my_l_table l ON b.uid=l.uid - WHERE - b.rid IN ( - SELECT other_col - FROM inner_table - ) - AND l.bla IN ('x', 'y') - GROUP BY 2 - ORDER BY 2 ASC -) AS "meh" -ORDER BY "sum__m_example" DESC -LIMIT 10; -""" - ) - == { - Table("my_l_table"), - Table("my_b_table"), - Table("my_t_table"), - Table("inner_table"), - } - ) - - assert ( - extract_tables( - """ -SELECT * -FROM table_a AS a, table_b AS b, table_c as c -WHERE a.id = b.id and b.id = c.id -""" - ) - == {Table("table_a"), Table("table_b"), Table("table_c")} - ) - - assert ( - extract_tables( - """ -SELECT somecol AS somecol -FROM ( - WITH bla AS ( - SELECT col_a - FROM a - WHERE - 1=1 - AND column_of_choice NOT IN ( - SELECT interesting_col - FROM b - ) - ), - rb AS ( - SELECT yet_another_column - FROM ( - SELECT a - FROM c - GROUP BY the_other_col - ) not_table - LEFT JOIN bla foo - ON foo.prop = not_table.bad_col0 - WHERE 1=1 - GROUP BY - not_table.bad_col1 , - not_table.bad_col2 , - ORDER BY not_table.bad_col_3 DESC , - not_table.bad_col4 , - not_table.bad_col5 - ) - SELECT random_col - FROM d - WHERE 1=1 - UNION ALL SELECT even_more_cols - FROM e - WHERE 1=1 - UNION ALL SELECT lets_go_deeper - FROM f - WHERE 1=1 - WHERE 2=2 - GROUP BY last_col - LIMIT 50000 -) -""" - ) - == {Table("a"), Table("b"), Table("c"), Table("d"), Table("e"), Table("f")} - ) - - -def test_extract_tables_mixed_from_clause() -> None: - """ - Test that the parser handles a ``FROM`` clause with table and subselect. - """ - assert ( - extract_tables( - """ -SELECT * -FROM table_a AS a, (select * from table_b) AS b, table_c as c -WHERE a.id = b.id and b.id = c.id -""" - ) - == {Table("table_a"), Table("table_b"), Table("table_c")} - ) - - -def test_extract_tables_nested_select() -> None: - """ - Test that the parser handles selects inside functions. - """ - assert ( - extract_tables( - """ -select (extractvalue(1,concat(0x7e,(select GROUP_CONCAT(TABLE_NAME) -from INFORMATION_SCHEMA.COLUMNS -WHERE TABLE_SCHEMA like "%bi%"),0x7e))); -""" - ) - == {Table("COLUMNS", "INFORMATION_SCHEMA")} - ) - - assert ( - extract_tables( - """ -select (extractvalue(1,concat(0x7e,(select GROUP_CONCAT(COLUMN_NAME) -from INFORMATION_SCHEMA.COLUMNS -WHERE TABLE_NAME="bi_achivement_daily"),0x7e))); -""" - ) - == {Table("COLUMNS", "INFORMATION_SCHEMA")} - ) - - -def test_extract_tables_complex_cte_with_prefix() -> None: - """ - Test that the parser handles CTEs with prefixes. - """ - assert ( - extract_tables( - """ -WITH CTE__test (SalesPersonID, SalesOrderID, SalesYear) -AS ( - SELECT SalesPersonID, SalesOrderID, YEAR(OrderDate) AS SalesYear - FROM SalesOrderHeader - WHERE SalesPersonID IS NOT NULL -) -SELECT SalesPersonID, COUNT(SalesOrderID) AS TotalSales, SalesYear -FROM CTE__test -GROUP BY SalesYear, SalesPersonID -ORDER BY SalesPersonID, SalesYear; -""" - ) - == {Table("SalesOrderHeader")} - ) - - -def test_extract_tables_identifier_list_with_keyword_as_alias() -> None: - """ - Test that aliases that are keywords are parsed correctly. - """ - assert ( - extract_tables( - """ -WITH - f AS (SELECT * FROM foo), - match AS (SELECT * FROM f) -SELECT * FROM match -""" - ) - == {Table("foo")} - ) - - -def test_update() -> None: - """ - Test that ``UPDATE`` is not detected as ``SELECT``. - """ - assert ParsedQuery("UPDATE t1 SET col1 = NULL").is_select() is False - - -def test_set() -> None: - """ - Test that ``SET`` is detected correctly. - """ - query = ParsedQuery( - """ --- comment -SET hivevar:desc='Legislators'; -""" - ) - assert query.is_set() is True - assert query.is_select() is False - - assert ParsedQuery("set hivevar:desc='bla'").is_set() is True - assert ParsedQuery("SELECT 1").is_set() is False - - -def test_show() -> None: - """ - Test that ``SHOW`` is detected correctly. - """ - query = ParsedQuery( - """ --- comment -SHOW LOCKS test EXTENDED; --- comment -""" - ) - assert query.is_show() is True - assert query.is_select() is False - - assert ParsedQuery("SHOW TABLES").is_show() is True - assert ParsedQuery("shOw TABLES").is_show() is True - assert ParsedQuery("show TABLES").is_show() is True - assert ParsedQuery("SELECT 1").is_show() is False - - -def test_is_explain() -> None: - """ - Test that ``EXPLAIN`` is detected correctly. - """ - assert ParsedQuery("EXPLAIN SELECT 1").is_explain() is True - assert ParsedQuery("EXPLAIN SELECT 1").is_select() is False - - assert ( - ParsedQuery( - """ --- comment -EXPLAIN select * from table --- comment 2 -""" - ).is_explain() - is True - ) - - assert ( - ParsedQuery( - """ --- comment -EXPLAIN select * from table -where col1 = 'something' --- comment 2 - --- comment 3 -EXPLAIN select * from table -where col1 = 'something' --- comment 4 -""" - ).is_explain() - is True - ) - - assert ( - ParsedQuery( - """ --- This is a comment - -- this is another comment but with a space in the front -EXPLAIN SELECT * FROM TABLE -""" - ).is_explain() - is True - ) - - assert ( - ParsedQuery( - """ -/* This is a comment - with stars instead */ -EXPLAIN SELECT * FROM TABLE -""" - ).is_explain() - is True - ) - - assert ( - ParsedQuery( - """ --- comment -select * from table -where col1 = 'something' --- comment 2 -""" - ).is_explain() - is False - ) - - -def test_is_valid_ctas() -> None: - """ - Test if a query is a valid CTAS. - - A valid CTAS has a ``SELECT`` as its last statement. - """ - assert ( - ParsedQuery("SELECT * FROM table", strip_comments=True).is_valid_ctas() is True - ) - - assert ( - ParsedQuery( - """ --- comment -SELECT * FROM table --- comment 2 -""", - strip_comments=True, - ).is_valid_ctas() - is True - ) - - assert ( - ParsedQuery( - """ --- comment -SET @value = 42; -SELECT @value as foo; --- comment 2 -""", - strip_comments=True, - ).is_valid_ctas() - is True - ) - - assert ( - ParsedQuery( - """ --- comment -EXPLAIN SELECT * FROM table --- comment 2 -""", - strip_comments=True, - ).is_valid_ctas() - is False - ) - - assert ( - ParsedQuery( - """ -SELECT * FROM table; -INSERT INTO TABLE (foo) VALUES (42); -""", - strip_comments=True, - ).is_valid_ctas() - is False - ) - - -def test_is_valid_cvas() -> None: - """ - Test if a query is a valid CVAS. - - A valid CVAS has a single ``SELECT`` statement. - """ - assert ( - ParsedQuery("SELECT * FROM table", strip_comments=True).is_valid_cvas() is True - ) - - assert ( - ParsedQuery( - """ --- comment -SELECT * FROM table --- comment 2 -""", - strip_comments=True, - ).is_valid_cvas() - is True - ) - - assert ( - ParsedQuery( - """ --- comment -SET @value = 42; -SELECT @value as foo; --- comment 2 -""", - strip_comments=True, - ).is_valid_cvas() - is False - ) - - assert ( - ParsedQuery( - """ --- comment -EXPLAIN SELECT * FROM table --- comment 2 -""", - strip_comments=True, - ).is_valid_cvas() - is False - ) - - assert ( - ParsedQuery( - """ -SELECT * FROM table; -INSERT INTO TABLE (foo) VALUES (42); -""", - strip_comments=True, - ).is_valid_cvas() - is False - ) - - def test_is_select_cte_with_comments() -> None: - """ - Some CTES with comments are not correctly identified as SELECTS. - """ sql = ParsedQuery( """WITH blah AS (SELECT * FROM core_dev.manager_team), @@ -995,162 +54,6 @@ INNER JOIN blah2 ON blah2.team_id = blah.team_id""" assert sql.is_select() -def test_cte_is_select() -> None: - """ - Some CTEs are not correctly identified as SELECTS. - """ - # `AS(` gets parsed as a function - sql = ParsedQuery( - """WITH foo AS( -SELECT - FLOOR(__time TO WEEK) AS "week", - name, - COUNT(DISTINCT user_id) AS "unique_users" -FROM "druid"."my_table" -GROUP BY 1,2 -) -SELECT - f.week, - f.name, - f.unique_users -FROM foo f""" - ) - assert sql.is_select() - - -def test_unknown_select() -> None: - """ - Test that `is_select` works when sqlparse fails to identify the type. - """ - sql = "WITH foo AS(SELECT 1) SELECT 1" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" - assert ParsedQuery(sql).is_select() - - sql = "WITH foo AS(SELECT 1) INSERT INTO my_table (a) VALUES (1)" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" - assert not ParsedQuery(sql).is_select() - - sql = "WITH foo AS(SELECT 1) DELETE FROM my_table" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" - assert not ParsedQuery(sql).is_select() - - -def test_get_query_with_new_limit_comment() -> None: - """ - Test that limit is applied correctly. - """ - query = ParsedQuery("SELECT * FROM birth_names -- SOME COMMENT") - assert query.set_or_update_query_limit(1000) == ( - "SELECT * FROM birth_names -- SOME COMMENT\nLIMIT 1000" - ) - - -def test_get_query_with_new_limit_comment_with_limit() -> None: - """ - Test that limits in comments are ignored. - """ - query = ParsedQuery("SELECT * FROM birth_names -- SOME COMMENT WITH LIMIT 555") - assert query.set_or_update_query_limit(1000) == ( - "SELECT * FROM birth_names -- SOME COMMENT WITH LIMIT 555\nLIMIT 1000" - ) - - -def test_get_query_with_new_limit_lower() -> None: - """ - Test that lower limits are not replaced. - """ - query = ParsedQuery("SELECT * FROM birth_names LIMIT 555") - assert query.set_or_update_query_limit(1000) == ( - "SELECT * FROM birth_names LIMIT 555" - ) - - -def test_get_query_with_new_limit_upper() -> None: - """ - Test that higher limits are replaced. - """ - query = ParsedQuery("SELECT * FROM birth_names LIMIT 2000") - assert query.set_or_update_query_limit(1000) == ( - "SELECT * FROM birth_names LIMIT 1000" - ) - - -def test_basic_breakdown_statements() -> None: - """ - Test that multiple statements are parsed correctly. - """ - query = ParsedQuery( - """ -SELECT * FROM birth_names; -SELECT * FROM birth_names LIMIT 1; -""" - ) - assert query.get_statements() == [ - "SELECT * FROM birth_names", - "SELECT * FROM birth_names LIMIT 1", - ] - - -def test_messy_breakdown_statements() -> None: - """ - Test the messy multiple statements are parsed correctly. - """ - query = ParsedQuery( - """ -SELECT 1;\t\n\n\n \t -\t\nSELECT 2; -SELECT * FROM birth_names;;; -SELECT * FROM birth_names LIMIT 1 -""" - ) - assert query.get_statements() == [ - "SELECT 1", - "SELECT 2", - "SELECT * FROM birth_names", - "SELECT * FROM birth_names LIMIT 1", - ] - - -def test_sqlparse_formatting(): - """ - Test that ``from_unixtime`` is formatted correctly. - - ``sqlparse==0.3.1`` has a bug and removes space between ``from`` and - ``from_unixtime``, resulting in:: - - SELECT extract(HOUR - fromfrom_unixtime(hour_ts) - AT TIME ZONE 'America/Los_Angeles') - from table - - """ - assert sqlparse.format( - "SELECT extract(HOUR from from_unixtime(hour_ts) AT TIME ZONE 'America/Los_Angeles') from table", - reindent=True, - ) == ( - "SELECT extract(HOUR\n from from_unixtime(hour_ts) " - "AT TIME ZONE 'America/Los_Angeles')\nfrom table" - ) - - -def test_strip_comments_from_sql() -> None: - """ - Test that comments are stripped out correctly. - """ - assert ( - strip_comments_from_sql("SELECT col1, col2 FROM table1") - == "SELECT col1, col2 FROM table1" - ) - assert ( - strip_comments_from_sql("SELECT col1, col2 FROM table1\n-- comment") - == "SELECT col1, col2 FROM table1\n" - ) - assert ( - strip_comments_from_sql("SELECT '--abc' as abc, col2 FROM table1\n") - == "SELECT '--abc' as abc, col2 FROM table1" - ) - - def test_validate_filter_clause_valid(): # regular clauses assert validate_filter_clause("col = 1") is None