diff --git a/docs/src/data/databases.json b/docs/src/data/databases.json index c2209b079ef..33662b38ea1 100644 --- a/docs/src/data/databases.json +++ b/docs/src/data/databases.json @@ -1,16 +1,16 @@ { "generated": "2026-04-17T17:08:35.132Z", "statistics": { - "totalDatabases": 80, - "withDocumentation": 80, - "withConnectionString": 80, - "withDrivers": 28, + "totalDatabases": 81, + "withDocumentation": 81, + "withConnectionString": 81, + "withDrivers": 29, "withAuthMethods": 5, - "supportsJoins": 74, - "supportsSubqueries": 77, + "supportsJoins": 75, + "supportsSubqueries": 78, "supportsDynamicSchema": 29, "supportsCatalog": 20, - "averageScore": 54, + "averageScore": 53, "maxScore": 201, "byCategory": { "Cloud - AWS": [ @@ -20,6 +20,7 @@ ], "Query Engines": [ "Amazon Athena", + "Apache DataFusion", "Apache Drill", "Apache Hive", "Apache Impala", @@ -94,6 +95,7 @@ "Vertica" ], "Apache Projects": [ + "Apache DataFusion", "Apache Doris", "Apache Drill", "Apache Druid", @@ -108,6 +110,7 @@ "Databricks Interactive Cluster" ], "Open Source": [ + "Apache DataFusion", "Apache Doris", "Apache Drill", "Apache Druid", @@ -9698,6 +9701,97 @@ "supports_dynamic_schema": true, "supports_catalog": true, "supports_dynamic_catalog": true + }, + "Apache DataFusion": { + "time_grains": { + "SECOND": true, + "FIVE_SECONDS": true, + "THIRTY_SECONDS": true, + "MINUTE": true, + "FIVE_MINUTES": true, + "TEN_MINUTES": true, + "FIFTEEN_MINUTES": true, + "THIRTY_MINUTES": true, + "HALF_HOUR": false, + "HOUR": true, + "SIX_HOURS": false, + "DAY": true, + "WEEK": true, + "WEEK_STARTING_SUNDAY": false, + "WEEK_STARTING_MONDAY": false, + "WEEK_ENDING_SATURDAY": false, + "WEEK_ENDING_SUNDAY": false, + "MONTH": true, + "QUARTER": true, + "QUARTER_YEAR": false, + "YEAR": true + }, + "module": "superset.db_engine_specs.datafusion", + "limit_method": 1, + "limit_clause": true, + "joins": true, + "subqueries": true, + "alias_in_select": true, + "alias_in_orderby": true, + "time_groupby_inline": false, + "alias_to_source_column": false, + "order_by_not_in_select": true, + "expressions_in_orderby": false, + "cte_in_subquery": true, + "max_column_name": null, + "sql_comments": true, + "escaped_colons": true, + "masked_encrypted_extra": false, + "column_type_mapping": false, + "function_names": false, + "user_impersonation": false, + "file_upload": false, + "get_extra_table_metadata": false, + "dbapi_exception_mapping": false, + "custom_errors": false, + "dynamic_schema": false, + "catalog": false, + "dynamic_catalog": false, + "ssh_tunneling": true, + "query_cancelation": false, + "get_metrics": false, + "where_latest_partition": false, + "expand_data": false, + "query_cost_estimation": false, + "sql_validation": false, + "score": 24, + "max_score": 201, + "documentation": { + "description": "DataFusion is a highly performant query engine", + "logo": "datafusion.png", + "homepage_url": "https://datafusion.apache.org/", + "categories": [ + "Query Engines", + "Open Source", + "Apache Projects" + ], + "pypi_packages": [ + "flightsql-dbapi" + ], + "connection_string": "datafusion://host:port", + "drivers": [ + { + "name": "Arrow Flight SQL (Recommended)", + "pypi_package": "flightsql-dbapi", + "connection_string": "datafusion://host:port", + "is_recommended": true + } + ], + "category": "Apache Projects" + }, + "engine": "datafusion", + "engine_name": "Apache DataFusion", + "engine_aliases": [], + "default_driver": "flightsql", + "supports_file_upload": false, + "supports_dynamic_schema": false, + "supports_catalog": false, + "supports_dynamic_catalog": false } } } diff --git a/docs/static/img/databases/datafusion.png b/docs/static/img/databases/datafusion.png new file mode 100644 index 00000000000..381265e62d7 Binary files /dev/null and b/docs/static/img/databases/datafusion.png differ diff --git a/pyproject.toml b/pyproject.toml index d379670d325..ecaf600014f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,6 +142,7 @@ databricks = [ "databricks-sql-connector>=4.2.6, <4.4.0", "databricks-sqlalchemy==1.0.5", ] +datafusion = ["flightsql-dbapi>=0.2.0, <0.3"] db2 = ["ibm-db-sa>0.3.8, <=0.4.4"] denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"] dremio = ["sqlalchemy-dremio>=1.2.1, <4"] diff --git a/superset/db_engine_specs/datafusion.py b/superset/db_engine_specs/datafusion.py new file mode 100644 index 00000000000..ec7a9ac4f64 --- /dev/null +++ b/superset/db_engine_specs/datafusion.py @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from typing import Any + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs import BaseEngineSpec +from superset.db_engine_specs.base import DatabaseCategory + + +class DataFusionEngineSpec(BaseEngineSpec): + engine_name = "Apache DataFusion" + engine = "datafusion" + drivers = { + "flightsql": "Arrow Flight SQL protocol for DataFusion", + } + default_driver = "flightsql" + sqlalchemy_uri_placeholder = "datafusion://host:port" + supports_file_upload = False + + metadata = { + "description": "DataFusion is a highly performant query engine", + "logo": "datafusion.png", + "homepage_url": "https://datafusion.apache.org/", + "categories": [ + DatabaseCategory.QUERY_ENGINES, + DatabaseCategory.OPEN_SOURCE, + DatabaseCategory.APACHE_PROJECTS, + ], + "pypi_packages": ["flightsql-dbapi"], + "connection_string": "datafusion://host:port", + "drivers": [ + { + "name": "Arrow Flight SQL (Recommended)", + "pypi_package": "flightsql-dbapi", + "connection_string": "datafusion://host:port", + "is_recommended": True, + } + ], + } + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DATE_TRUNC('second', {col})", + TimeGrain.FIVE_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '5 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 5)", # noqa: E501 + TimeGrain.THIRTY_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '30 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 30)", # noqa: E501 + TimeGrain.MINUTE: "DATE_TRUNC('minute', {col})", + TimeGrain.FIVE_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '5 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 5)", # noqa: E501 + TimeGrain.TEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '10 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 10)", # noqa: E501 + TimeGrain.FIFTEEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '15 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 15)", # noqa: E501 + TimeGrain.THIRTY_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '30 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 30)", # noqa: E501 + TimeGrain.HOUR: "DATE_TRUNC('hour', {col})", + TimeGrain.DAY: "DATE_TRUNC('day', {col})", + TimeGrain.WEEK: "DATE_TRUNC('week', {col})", + TimeGrain.MONTH: "DATE_TRUNC('month', {col})", + TimeGrain.QUARTER: "DATE_TRUNC('quarter', {col})", + TimeGrain.YEAR: "DATE_TRUNC('year', {col})", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "from_unixtime({col})" + + @classmethod + def convert_dttm( + cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None + ) -> str | None: + sqla_type = cls.get_sqla_column_type(target_type) + + if isinstance(sqla_type, types.Date): + return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" + if isinstance(sqla_type, types.DateTime): + dttm_formatted = dttm.isoformat(sep=" ", timespec="milliseconds") + return f"""TO_TIMESTAMP('{dttm_formatted}', 'YYYY-MM-DD HH24:MI:SS.FFF')""" + return None diff --git a/superset_text.yml b/superset_text.yml index 3b3a346b54f..aeb84d631b7 100644 --- a/superset_text.yml +++ b/superset_text.yml @@ -19,6 +19,7 @@ # To set the images of your preferred database, you may create a mapping here with engine and locations of the relevant images. The image can be hosted locally inside your static/file directory or online (e.g. S3). # DB_IMAGES: +# datafusion: "path/to/image/datafusion.jpg" # postgresql: "path/to/image/postgres.jpg" # bigquery: "path/to/s3bucket/bigquery.jpg" # snowflake: "path/to/image/snowflake.jpg" diff --git a/tests/unit_tests/db_engine_specs/test_datafusion.py b/tests/unit_tests/db_engine_specs/test_datafusion.py new file mode 100644 index 00000000000..0595f885cfd --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_datafusion.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime + +import pytest + +from superset.db_engine_specs.datafusion import DataFusionEngineSpec +from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm +from tests.unit_tests.fixtures.common import dttm # noqa: F401 + + +@pytest.mark.parametrize( + "target_type,expected_result", + [ + ("Date", "TO_DATE('2019-01-02', 'YYYY-MM-DD')"), + ( + "DateTime", + "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')", + ), + ( + "TimeStamp", + "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')", + ), + ("UnknownType", None), + ], +) +def test_convert_dttm( + target_type: str, + expected_result: str | None, + dttm: datetime, # noqa: F811 +) -> None: + assert_convert_dttm(DataFusionEngineSpec, target_type, expected_result, dttm) + + +def test_epoch_to_dttm() -> None: + assert DataFusionEngineSpec.epoch_to_dttm().format(col="ts") == "from_unixtime(ts)" + + +@pytest.mark.parametrize( + "time_grain,expected", + [ + (None, "ts"), + ("PT1S", "DATE_TRUNC('second', ts)"), + ( + "PT5S", + "DATE_TRUNC('minute', ts) + INTERVAL '5 seconds' * " + "FLOOR(EXTRACT(SECOND FROM ts) / 5)", + ), + ( + "PT30S", + "DATE_TRUNC('minute', ts) + INTERVAL '30 seconds' * " + "FLOOR(EXTRACT(SECOND FROM ts) / 30)", + ), + ("PT1M", "DATE_TRUNC('minute', ts)"), + ( + "PT5M", + "DATE_TRUNC('hour', ts) + INTERVAL '5 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 5)", + ), + ( + "PT10M", + "DATE_TRUNC('hour', ts) + INTERVAL '10 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 10)", + ), + ( + "PT15M", + "DATE_TRUNC('hour', ts) + INTERVAL '15 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 15)", + ), + ( + "PT30M", + "DATE_TRUNC('hour', ts) + INTERVAL '30 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 30)", + ), + ("PT1H", "DATE_TRUNC('hour', ts)"), + ("P1D", "DATE_TRUNC('day', ts)"), + ("P1W", "DATE_TRUNC('week', ts)"), + ("P1M", "DATE_TRUNC('month', ts)"), + ("P3M", "DATE_TRUNC('quarter', ts)"), + ("P1Y", "DATE_TRUNC('year', ts)"), + ], +) +def test_time_grain_expressions(time_grain: str | None, expected: str) -> None: + assert ( + DataFusionEngineSpec._time_grain_expressions[time_grain].format(col="ts") + == expected + )