From 7eceebce1ceaca78b548e705ab3630fa1e9dd687 Mon Sep 17 00:00:00 2001 From: Bhargava Vadlamani <11091419+coderfender@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:56:30 -0700 Subject: [PATCH] feat(dbspec): Support apache datafusion dbspec (#42374) --- docs/src/data/databases.json | 108 ++++++++++++++++-- docs/static/img/databases/datafusion.png | Bin 0 -> 14784 bytes pyproject.toml | 1 + superset/db_engine_specs/datafusion.py | 92 +++++++++++++++ superset_text.yml | 1 + .../db_engine_specs/test_datafusion.py | 101 ++++++++++++++++ 6 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 docs/static/img/databases/datafusion.png create mode 100644 superset/db_engine_specs/datafusion.py create mode 100644 tests/unit_tests/db_engine_specs/test_datafusion.py diff --git a/docs/src/data/databases.json b/docs/src/data/databases.json index c2209b079ef..33662b38ea1 100644 --- a/docs/src/data/databases.json +++ b/docs/src/data/databases.json @@ -1,16 +1,16 @@ { "generated": "2026-04-17T17:08:35.132Z", "statistics": { - "totalDatabases": 80, - "withDocumentation": 80, - "withConnectionString": 80, - "withDrivers": 28, + "totalDatabases": 81, + "withDocumentation": 81, + "withConnectionString": 81, + "withDrivers": 29, "withAuthMethods": 5, - "supportsJoins": 74, - "supportsSubqueries": 77, + "supportsJoins": 75, + "supportsSubqueries": 78, "supportsDynamicSchema": 29, "supportsCatalog": 20, - "averageScore": 54, + "averageScore": 53, "maxScore": 201, "byCategory": { "Cloud - AWS": [ @@ -20,6 +20,7 @@ ], "Query Engines": [ "Amazon Athena", + "Apache DataFusion", "Apache Drill", "Apache Hive", "Apache Impala", @@ -94,6 +95,7 @@ "Vertica" ], "Apache Projects": [ + "Apache DataFusion", "Apache Doris", "Apache Drill", "Apache Druid", @@ -108,6 +110,7 @@ "Databricks Interactive Cluster" ], "Open Source": [ + "Apache DataFusion", "Apache Doris", "Apache Drill", "Apache Druid", @@ -9698,6 +9701,97 @@ "supports_dynamic_schema": true, "supports_catalog": true, "supports_dynamic_catalog": true + }, + "Apache DataFusion": { + "time_grains": { + "SECOND": true, + "FIVE_SECONDS": true, + "THIRTY_SECONDS": true, + "MINUTE": true, + "FIVE_MINUTES": true, + "TEN_MINUTES": true, + "FIFTEEN_MINUTES": true, + "THIRTY_MINUTES": true, + "HALF_HOUR": false, + "HOUR": true, + "SIX_HOURS": false, + "DAY": true, + "WEEK": true, + "WEEK_STARTING_SUNDAY": false, + "WEEK_STARTING_MONDAY": false, + "WEEK_ENDING_SATURDAY": false, + "WEEK_ENDING_SUNDAY": false, + "MONTH": true, + "QUARTER": true, + "QUARTER_YEAR": false, + "YEAR": true + }, + "module": "superset.db_engine_specs.datafusion", + "limit_method": 1, + "limit_clause": true, + "joins": true, + "subqueries": true, + "alias_in_select": true, + "alias_in_orderby": true, + "time_groupby_inline": false, + "alias_to_source_column": false, + "order_by_not_in_select": true, + "expressions_in_orderby": false, + "cte_in_subquery": true, + "max_column_name": null, + "sql_comments": true, + "escaped_colons": true, + "masked_encrypted_extra": false, + "column_type_mapping": false, + "function_names": false, + "user_impersonation": false, + "file_upload": false, + "get_extra_table_metadata": false, + "dbapi_exception_mapping": false, + "custom_errors": false, + "dynamic_schema": false, + "catalog": false, + "dynamic_catalog": false, + "ssh_tunneling": true, + "query_cancelation": false, + "get_metrics": false, + "where_latest_partition": false, + "expand_data": false, + "query_cost_estimation": false, + "sql_validation": false, + "score": 24, + "max_score": 201, + "documentation": { + "description": "DataFusion is a highly performant query engine", + "logo": "datafusion.png", + "homepage_url": "https://datafusion.apache.org/", + "categories": [ + "Query Engines", + "Open Source", + "Apache Projects" + ], + "pypi_packages": [ + "flightsql-dbapi" + ], + "connection_string": "datafusion://host:port", + "drivers": [ + { + "name": "Arrow Flight SQL (Recommended)", + "pypi_package": "flightsql-dbapi", + "connection_string": "datafusion://host:port", + "is_recommended": true + } + ], + "category": "Apache Projects" + }, + "engine": "datafusion", + "engine_name": "Apache DataFusion", + "engine_aliases": [], + "default_driver": "flightsql", + "supports_file_upload": false, + "supports_dynamic_schema": false, + "supports_catalog": false, + "supports_dynamic_catalog": false } } } diff --git a/docs/static/img/databases/datafusion.png b/docs/static/img/databases/datafusion.png new file mode 100644 index 0000000000000000000000000000000000000000..381265e62d7bf3df77cc6c6216e11755f9afdd59 GIT binary patch literal 14784 zcmV;xIX}jUP);>c4Qc1P}IF>Crk_gC>^;jtw zSk`(iWkoGfAHcU-9UD;b1FZ&Tk_{U>6)oj%8L!#`iC#%0V4Vz&sBc( z!?_GR@;rSWYuJ_&*XVB!RWv~#=L9^%wESD=$DfGmJVB>O5Xovx z@ju2D@hJx|)J%(i!N!W>6Jr&KtMC<|*-uM|5kzkL;+QmXvjyMu0@fKntqa~!3Zsr-+- zBOCpO&Y7moK=8X99UHDa@0sId1h8>}!7S~bHV6XCo$QXXpt7V4X25VQWEQnZpDVy` zyTXe8Z1pwe*qm+vtw_}eauQT&^;^W%e(&k$YTKX!0jm&v9)cP!#`Mf^GOc^5>NUba zK_76E&f`GqTox^0Od{q+0xATa>%^zZ$H{`kV!g7f+dfgv(@&nK76u7o1!cLQZpHys zkO*Ci3k0-*?oYJ0B}%;dDviz%b3FQ34!IaAEQK8HnE=^xu|KyQug@xe%iMzp#+S0- z0`Tn00*tK8l9cLd_m^&!w`$)4uWbKMX&p?)1!`uT0G6}roC9)>fOc>7t-{E$PnBn& zO(_y^MM}5`mYYeGq=O;@Z30!=X>sVq)@X2}A*w))m9ftsrKAM4YH*#oy%htXMq9c*$Mb3&0BQt33je39O(WEfvdQ8qKv_&AR3@6 zl#(7eT~HO<8E9k_l)zY`IsQywyGq9@wdWmLkS%|qpMa$SmQ9p*l_=+$3^$V`;<{DG zneO7wIM-IhobA}y%@MLS$r8hg1G#Z~KymG@S1v=F61|wt`mFq~AH7#>bk3m!83RnB z&;WBH(mv2Y{Q~O?Q)4~FTceE>`n{&^v0G(wR1T0eG3T*PI|TB{wQs*N1Z_yfxyXm6 zE|Ht&(njYTI*_rYCxHM4m@@~=9}F!$y;X}s;K1?X?MY}MF<0xleB%st=&I8XZGR@$ zzH=@QZAg{BdvAurM;m>A=wg?TKY#8TJ-9*l{1OMMb*2if>@TWF4fGW6%)tTD@|E*N z3c!#1ev&9v9ir{s8?|((IN9%F2QmHvpPb;+UxIGg)iX>PjIChk)81WDh@cZkWX65ClPOP0EIh zjN77eqAOE`r!?^{pp(} zNunKWI-9`g3bZ-3Ey$RQarS9)1ZzS@z#_A9^6Phs&}P&v1?!jvb&V{jb`Ru&T zu)rjv*^!-yTRHLO?LIg}M2i1Cx>KE@p)yfly9LnOijysUigYoBIsHNT%mvN8TnLG| z7#uR95Yw2SF;#6p-K@RDCF12UrNU%dkRXwEQmw*D&o}Rs;E<7xVe)^T{PIifW@)y~ zY^6t^>@T2_&xd6J{IoN)a65!t%#p%qLVzsLZkEGMWCTn_5oF_4(143YoQp)nd*r%_ z7Nw-{6g}NY!D8G2i`^p!Ct_Qold0iC_AxxpNfGhf#M7cf? z&$=xL3?fjAw4JjPlIIAG_c^ZHIa1!&+`4@M8H4H(tPBc3<~#)gNxt^x{XSZUD(6WE zMCE0nndYzjEH+I4zJ5R%h;wwU`}Dt}gqrKAcFp1B{^#!RhL(Sa+!CEUN0Y`%$vlk# zDfsFXPeMCHZRl*RpSWsMg$%!@oBWkYb@Dd~w(jR)u)FKVrB1U47on7F6i+Rt*YT9VD)y-PK z@|#PWIRf2{PFSDo{KgLtEF)HPsAJW<$`D7A;#DPpIlI0u&$(UFrLFWr2j17y^?2%; z&=nB!gfa$Wh2yfm@km?RS<*|he4apcnkO@(;Brv9T93Yt3dY{1qJzb$=XylM+cn4@{=(To>eyusEM0U2{M(k|K%j4sfW$F5D8~Tr6ACIF9cpf+t81}$6OOa= z>*TgT%qa-kOuH^-2t;MNKp%bJl15q0J%c+8GG^)eYK|Vp42X^!`yCZn``WpD6?Jst zRq8#9yKI9OgU5uxkMDCqy*Q5NqK?kRMrEfsOM^DkxXla=HBA7oGN@zfgSbf7%p35b zd1hd|+P2<@uiijAB?8-30$Uk&NIL=NI5}BA*h`}?B#KgR$YG>TcRUZQvBPkig0A8We3&z>*PuSwjf zi%O?EtE|&qNE9s!dMW6BX1Y2$t~dMeb(Kbl2Q!g+HCFmzWxf6*SPg)gSi+4+>`lqf z)wj-%Dkc%i>XTheQt(;!>;a6fed zrdj9GPgq!O)2f@At0>L2CQkmz?l}U}^2hsgG2*;(T+wm^NQ=vnp0{@H8+-JH^93^V zpC`+0P@vHzopnq1Ged(r)BUyER~z=Pc)kEa4M~ZtGGr|mU!rQH3gn**>9v^rfV0)x%I z({9WY!%Y=CKKk~=R_$yFvZW8bbU9Wy34t=Gl8bUdxQSEWe|KLp9v*%C>?~;ZjWs%g zH^s|6-+pIb2AJvbmoE$wqIjb4t6E_PcWwQ5k>2Jq?M`*U_VlVf>Dv|7RpA=@5s$@t zBQhHii&5fgv%2I};70N~EvXinRbX9h!|QJvRw52IPNKj7C`{PWksUUn@yvb!j725C zskO6?%;g>0>&q1FB?(w3(I*!4rBqU^#Tw38rPFY#&}x7&9fzol#;KZJ?~t#;J`uYx zjt$@Fqj8u@>UUV@CGSc#xZbH|*?QfmjSWGj0md1bwo%u(`A>@d^070$Kd3Jgv{$qq zLm?@KXm2Pf{A3my5O#FH)ORZ z!4m@8NY8h7pK+l^$* z+!}2bU`z}oEsED0#*bNB2_)OLa<62qvxTr}Z(?346b{v%WC69)ITC z_~U4OUXqYBZHtK$Cp!90e*IQ)w@Q3>@n)%eoWTYI^uJ7fwqU!kPhcaCY0|UzP5G}S zwj^VTV__jblWN5q$gXulDhiPjmy3fh%_Fn>fQ@KxJ}J4U{@vX^Xb~-a?CiYu$(cxk zT_yyQ9#io4N9$hE@>kCnB;YVzbI8VBOXS?! zGcl^eOu~ej6uB~5xy;VmA75nW{IEZeM0}uQ75ZFMz;m*N!G0PD;$oqx4hE!>D>%1^ zIGe-I3Ri)x9FmkXunpqLCj_v2^s*F@>yuHG;KVm?_w6^pn7D%WiLcxo#<)r^sk%=0 zGz=ZY*?wnlIGe7a!J>=b#VBr>$YhFRV{N9$f~MLD-rDvcqro3{Qq{W-%=wkxKigYR z{${i;jBm{KH|Yy{KWw$Dt0{fy(P)2R>GI3$8J||#uLTpwRp*(5NYU;t-mM(}@@;bN z!T<-GA%sNP$CT^H`#9ih@M9r?1JhZd4f146A57zzMKtZLNhW*TI~#aPkKrI?1D zku7*iRx8P^=Ha`c`=J7ztlYarA|jVCSo+l2SyX}m#wpt{LE)ahlaqIf2c5*ooUQxI zw*~;?5zbVlJ1Q=}eqnz3YdVEzN2Fxty7phSQ2BQxfF~e_4s*B28D`^S#z`#fiG$|L z6cjbZ(oA4z87a7ke(F&WCUYKU^3!ejQ&#Ffd2YN~759K=m;;v9h5GTQ0bI4z{o0*U z_iMNNz$_-g4`G@_=)BI9-ELb(4_jncMypq(i>l;H1n_C|MuoPkN}(w5IYwMcc`;F!u`!y~t8V%6xO)6eT60E+qO|a3(%x5vcDOs1f7C_QiykcQ6 z&z4p`2_L!43S@=m>vJp%`wD`F%E!yY#7fH*oK>JgZyQBT%?3E&~n22R;vLYn$d0*>{Z{V06S^X zdClTn#b`T+QKyUn-h&oWRjC1&0$dl6H{vsuSxcd(%T*NjZ1}0v)6v;?{JZZ=L7S2W z+ZS)M>M~-cHjoH?&rE8Ty1#g{q*OFOe5u5S z5U?}%8=G69UM~ZcZ}sHgSc(VE-Nlpy(0SrDzxD`dT04m+3JUuw;U@;=T>3J z0%tY%>OgvP*b0TI^{&n{C%@M|qOn2ixyUU8+0w9H@+IpHU@tOL{Th|An&eqeDwYE$ zfB(*C?;+t{8&T(Lcjmf3f0K2!BCwk&6VI=HW5bo@=J@55O0-mD>I6<7@&g}c(qNOA z4PuwcB3CL=zH`tbBCd7XNc1r<>EAdNA3t{Hc0#~NM#adnO+0*~5aj!Fu0e zdGnPktn3HMe-2tm1fpDRYL=5e`gtlOm+i^76I-N|} zN`l}+6A%DQced#y!e@oF1^PK&=bRUpr^MnFBQH zfmjFm0D{nsO$XA1x?aNm(*j$T&TWyzC=)W>ekS#>W~lpfw~E9NMx;*fB~eM64uPkj z1;m*aT0e9Oj#9yMr2ROW6A^S64xwC-6Tq!WMCWD~24C2BpmkK^i>?nwV>IkGbj;R< zla44&EIu*HoKc{Qx3E<=+6s}$MO=8RytuSXLgNV(8IOwsI-gWR1=}nB?gz0bz&_HW zU*M^CgbskuuscVxKkGR-2NN^oE(ErD*WSU%^+AvD6W8%dtlvH`N%z?>Sn7q%p z(*5B-vCRkal<+x^Oq}9kcPn*1OZLhL4=9K4=20Hu1UwN7Ou_oY&h-HOCv)(|ApDJC1{ED0giT zEdODl0AQH|1WW;ZHnY!Q_5pIGXS^gb)BgbMR7W9(VTZKz^Zi4Sh&b<_d@8bd>QCPp zb>Fc85X8H{#wi0wtnNbvj%XW%?;9q8Id{YWEiw@Tkk2FT&hkVRkE5SC9%Lfgz4e%9 zwj1&wlbx$qAz1M?mAa_(gCE+70kHJ5{X?Ylm5lcnCQkj}y@~Mqjw*h4Z>>t zzD=dCV+stkWrgfZ;{-h9%8FMfK zXUQ726Q>#gOgD`(TR%AzgTM3wl%DPxM^78Yap$5=aES19qwhJBW0^Uc2AaxY)I88M z7sF(Bxbv@uKGs-ZN?N5r*889Cmi00-_T+lNVHSFxY{|`xh-83iF*|YU&l&^FG%2#) zZGoY<2G5F*XnVBMrF~?6y^Thc08RTS4bYlcv{mQ@Z6S2HCS<$=bqh??m5xHL4;xHL z^9HGHkUYGX7Y6%tumLf^$m>wzU&l&r4lr<;9x4%)F3-n;OB^Zv+a?B(_Wt1Jd1dB2 zaf!ye!kHAfzB$n7y8=C_Vcz!+h#DzG&+}^S@9ID%yH=(E6A!zT63pZP+X&BXjA0xa z5C_v^DJS*s$Pd>0=kFC^3-W7lIb+tlPRwBmN3lX!xk&-ULvz_L!hCO z)0nNurem23rb2P)nTu_VLQTU9AM7uLzhmx3zP4FVA{$F}zxUeR zVQ7M=t=@p`dS%ReP^7BB5pA2S8z|Szxf*Cv@GSNU=cS3Ns&m+`d{4(ebGljOd=CH# zJ5a>L8ZQOexG?d`+9cRekIz;`FF5U1`49yIjQkC=)EqB?Q=x7bpjlE_Z%8DeOT;6U zY)6#hFij8xttzXR=$FJ~RDLxR`$EH-v&E*col@as5lhA=_kJTT#?%mGz2!Nc;xy?~ zI8^dK;Bff|AAE6qVX%-zskl5-j*fE%z_v<$y0yMQh1}AHn2u!Sj%g=RlAf_Ua|*?n z)!+*&-?p*Nm|PaDvpKyh(3!pdjvFs%yJ3lS*|j#spz+z-Dz!2Dh{Y&gxkg~jX{?~U zwA}(NDqNDuUcifK#+t+U2kyz6hrRnUo;>Oz+J0*OH{S<8;`pJ_e4fX}87^$aX-+iN1Z1 zF?C;lyNKWx;xL|l)2P?SeElBn7gIR0S$mtVNJ4(I5-7L8IwVfn`cuoj|8eIllcYQs z_6k0c@yNd1Ic-BHtEDF~j`@7P`X_EZ{Q3z^b$?>>I<=dEOt%xuFLc(Tu4hg zOCwg-HgnQ$N+COjK;c(M2-`79?_1b5tzNk>NY9|C>xi+(6ViG)lU~X*g5p+OjA=`d zY3f>;1Uo;|+4%_ zk~pu7oK&UGUwiv%Gi^5Y1kGfVTIJ50uqs$4F#q7EU)*+zZ|KwUoY}`Qk94w~kW>O; zn@&b{z+#u2Wp@SC1ic}ZkcjU~5luAPZu4wnMX!oh>G|BP4+9AiAZM!Lu zE&P)g$8{Xc0D#l97aiC%Ta`XV(t{6U*_g@YjpW7UW##R+7e_71&&Ml1(_-wc2G(LYZ@Xrh0GwQ33AX@*h$MyxP+UfGc|&=pqx-!0L^n}Oez0Iv^yfG zWVaf}rLUjMSA`_cf$;g$7$%Oom>+0kb&P}P-D9`P^r$B7bIz?~5ABn|@-*v32sBYL$r%40U;{jM_~>h7!kZhgrTC zYrj{=RxSbP9Hazdfl1@*jB#n9=Z?)bnjb(u!w2RX?yrwnI0NGdWIG_mBfCL)2lj;t z8C_$5*)$z=A*3+3%I?i{kG)+Cf6vQM^W0f$X=X9jR_TVcA;=a!@ZvaKKW9Tb<>&tM z-K(|dm;^`U1s)#QJM*0jM5Z~0n8#Dgvz!_fw$Z_yOeO96z*pABD|~fQBH`%D!F*)Jy6O9dE3onLKeY zh(PyN1XhXrj#sBpX`rpghLeH&In)6w2S>DRQs!Ay!eb6#q~_$LO!VnQHin2JMW5?v zj5W>;G{{9qb@PJ2Jj;pB983D^HFy;UjBJs;6wrKy zZD(ON@R=Ra_Gr1thE{symcDi_uT*@G6rUV_#+2t(e)R($S** zO=0;vjn{&z0~vF%Sju5<>BZHi)+!=HU{VLtZKu)RMNglI*S!b)%Xzs*3C)caR=i%zr}D@BF9fiPUbIvQ8_shTe#`6m30{n!y7DgQel zCDenpuf8%w0Q(y@piKI7u(*+D28mT4NgU}JD>t2d+F2D-hMQ^Z<{p2yFj#v|J;+iE zmv5|#ZA1(->UN3uosJcX(DYX2$Y{Qn)VG4bM-%KL=HQ5$CxO;r!@(-Z67AclNvI=3 zAU^Ez7)FkL{$|rmMYpKuZnC9m+K8m8tLm5n+5at&Uo2F#cerZrslR%6Bm|mwM|ekH z%8twi`;gKltou1q;)s)Ni_~eWPhU)ji0p(Q`!Jkj0W%YLgjc779Z)2zGhPIPs2Oo_I(wCF*L`) zt>G?GV{E{t3QkjKbNufN7Up7I@#X{~0RLeAM1$`h;Q-W97lYJh$0$9=*&LkdEEQB#l z53&d$j6p4KN8#U$PW|P(Bb-I)gRe@Z^)%0B{Y+%jXB&7ooLu$65p7CKMRrlF^Ypd4 zrHaDD1r@%`3ibwIM8~S)s$&ZLOjkSQkoztH8JIN^&2z^au!qDzQ-Kj-M7LCKqKISY zVZ1Ckk++RYARUpR<$w9|5QP9|!RpgSB!=}%YK<;f6fs=ter<06rmtz7wl}<#R?1tM z8f5>c@Q(_{`RbDe%dp4P`xl9O!{EtW$>Yc>rpBPQfb(4KNGIEfo_yuyD_lj1n10q} zvUtN0RG{~oEzzPxIdJSNH}^zRg`}BN!YI7R)F8`%(ABM-nj}D$VIRl=jXh=o%Z6IZ z(oI%DS6)=9Y;DZ*StqT3efeePUZdXGog#$r4Ypyv3RW{`dS1J;?Mc{8k>IQxyRH&l zZ?z*H)`v}h5``bMKK8@?=>!;{Nhy9=r9v4Bum)&hemd_(9jntvo_|BjUp`-u)paiP zT!7m6FoaGZ`_h_0u0R|ObQhcR zFS9{;QyRa(yP^u2d}`*DYEA74uA8AcI8DgR=S-HbL#UrXtei79C#3k>M?c>`4$l}Z zf1yA3_=_*!qj4DZFC$=DN~-j@vEC~Ir;ZnI4`8uX3!tg{t}mo3X4kG1@*)FK+T<0j zTVjSQ04cG0hE%Ua2~}n(Lhmv3PDAUt=7E!{_~7UI=bshv`Q-CwFULwS!18HOnKahA zsc+VZA`Fi96mRWldg4w=O3TTd*Pt_|7&S`ogVM5F+U25KM4b4Tz5ebcfmjNR0CeMI zduBI*mBjN0I@G-fA9|^!gV;)19_`OP`P|F1=%>FsB{LDknlHa^Bfjfj2R)WJGXQZ zBrfwx0qP*e6CG6PiAc;jh2NDuNK$_ggNzIpFi37hLSaoF4z8xT*Jp`)={zk}lROYu zk1G9*eIqLtaqq)HQEK*?;%e#76QnQ~iuCw^W^A*5#_NnL?`QF-g0y00kA<90Z;{sn zxbombq^q?lS6lwfx$!s>_nfs~UpmXVcLJc9@mK_@Q{XH56W8WHC=!ugOmVjhXd=y2OsVqroex(=lk_Dt)2!_qlN4fSi52p zMUxfz+}3k+r{449K_3O7u!;EAi8qY($N_S!@*GMyI4-8UC3+F1*JjLPLr7FIvy+1Z zBH}=^F7HW`N`H^u$><(MqNqXQm&71HEypX=Eq&zN^wVY-JpS~#@yBXyo;p0N(r>;O zGl`=$R39{v^(a2j^G4vFSY!8wuB=ps10;ni4?&8zN!8+jsjK8-Jfa8YQIDh58=)w4 zYOb?5T@XDG6?U5dPWcCv=q*yo;i@1J`q7uJ@cQR)fW)Ga05(t0p$IxFSfxaxN;y;A zP69T?&eiUYGbi6ji$vu3v%Xh#pi60!5IOB%3age0Ua$*YLWJdHJYxwCfON%eUKInJ zlt|u&z;?e%BROdWBzVD+$kxETEF|7R<} zw*2w_+@pU%-~utL1KOb~5Il>p)Pn<_L?qg77*4Ca&THXj|EfHv{gBI}eCp5d_K}J? z0jf6dz)(uJ3*F*o+ef}A2HX6@UpzMqdrSRCaR#3^{>^ zj-!ztv*Pu}-aqHmA?O2Do6%A8I0HReg`fknrj?w2_=^m-{$bc9F}NOn{A{TTIL}sN zC^-U;U#efGzd-^g`Mo5Fi4$+$-jI8RBoymo$m_)Kc_L!7q)Q+uW5oDzCqR*;ER*ZZ zJXCu$$yY@+4hKM{wuWAR(E>xIAB=YHrNucra!>vHcl!uzBMOX8=R~MuUgnlS&yr3% zGM$TrtZD_*4}axB3_*o%U#M5_Fd`A}R|z?X7H1wWDsFW(UU2=%u{gXqp@* znpUCF?})3F^M;|*ot02Rk`eENT zCRagQ6C^cj3x&Tw`}XVl6Frl+2f8Pl&-?0JB0b0i;VRv4>~Fre;E#nqIlfw1mQl=zVk9~Gv?#k6k=yF{GEO4XrMc% zX?qD;Ntu;7SnuR4%dD%_U?TvY50S`Z<79H9CVGFMIrPB7&-D)rp;sEtrR1&%H6+L6 zt-4?Vz?n3($7PmEv|U&lK6jm>2^H!MJ3-CM7a$jXQOp`yiHY%UefInlR^+F8GM$q+ zx$}fV>fuHE3SET*pw(}@GEU<&3`vJP$*Q`VI`sbrUM#ULfsKqG3eEKX&3hBjLR$Q( z{#oiv!MN`L+`^dpnVLS9v)A7pXtuqF6k?*E^CIoJ(7%CUDK%&O_%9RiC@TTo7E-Ro zq&?p@I**Am{na(RZ1^9cwB8{ndM9^2L!arl^3CfL=R*pUCL2iSY!fLC?V>u*R~a&*iF=eLdf~GqPKNF$omL zz(92aT1?AdJzt<_G@Tp?2TiLAK4H>>Od9~oiXW@NiE!{3XoVNAwMs{mnN`DTT|%WZ zM`R%4vdhD;O(HQ`Bm_A4HaJqh?vn3$oVlGd9`@)qNWGPwE*|}qa zFjzH{(&1Ael``HFEMtMU6UvrHzjc9_~&odc_MJ5#!3Gw(zoO0Bpz0``h&H$ zAT29+w$Zv&*Z*Gmy*<6*#h-d<76RvMka%Hvj4D4@V`UodwpyTYMuEP6o+o3N9|vvt z&q{gaGi4zf%h0$nl>Je%MdxC3RwP;(&L2{MP1;2JLuKH-h(gW960VC#&7J&q^PqeD znRC;$8K$P@kH2~Q$6$|1i$#bFyby#RJ5a%x^UcV_v^HaLzk4+*G>Wm}W)};ArZ1yv z3@LZ*kjoG*sAJ`blu?e0?zZH?!Uy{cbU(AT9Kt_*)2}fC6d%5*Ov#0uKi9gO!(1Rt zBx2x>;5KU*cM6OUfD{43Ha0j90*>b!4csFBx=y`8Ri^v8F9pT>)~c1B?=~Jg0vU7n z9R0X-;+wZeU=N6a^*D|PBOxy`$48#T;$J8M&XuGZyr}fe{qD8K0JEoTPPIU5*g@rA zeAX3QQDV1H4_}xDOWch8)bjg!R`noFUaWe?JzWI7b|% zLf_BvOWPsv`Iszy{d>87re|{dFEJNuk*jfDWn?ZA;&~}$0ky6j_C4u?>woexfFEsATY`*sxNC3;eLtX8Zh9roYpHS=8mjpzz>qhR2Jn+qRu z4OcSEDEw{ER?xZB?=g7rj;opT)FL&;I^T_xe_}-10)?0;R5GkM5KbHgw*oNM4VKx6 z1bpF(5xzL7euuyX{je92|8xBbG<{VZrs5C6V4fH9qA}&%f6l_qY7Q9o0Uc=k19E`! zOr6EQ_QyEWUU*;SWY?E(j=>hx6v&v2C83L_ICpdffDp}{pE7(b_CCT(~@`Cm`S4XU_7Q?0&tbcp8B8=v{hI)-K0U04g!WkV%>>K7R9&;jK9ms-x|6tM_ zs6s!Rp$4v>{K3vFKMgp9Vd^gFhF1JIJ))lLCY%8=B`WoOYyiGGF&+RX98a{|%#61* zTvH|Y?%D`Npl{D#v*v|0pC*p0%;a_oxNI9_jut-9KTHbw5ZzFrzR`LysXM5kMq#wl zpq7i-@BKC${FJg|&?(335ST>-(XTUwk*^kJ3AZ0YMlk!tL2ALzv z*+zj2p7Gn;?1iy`G7OL?ycr2VPmuszOI zOS=YHNMwmpR32ZIW}1C&%O8ZmGGf=f)G37@5;uVT4O(LX0*>i9S#n;5{+6XqFXw6d ztpn9o#NPjPiFtai{9@*r($fV}*BoN8w^l1hifv}gba?0L(POs&QT(rYX3?W^g3&lJjU(o__rT0NZ zIXX1%-gM_IbZ@Z*>*%Tf@NUa%bL}-PkeKbPu3e^kD%$axLoM=Uj^41W8AlG1+iVES z%|Z^v9NoUW6=mi>&`SvKOAwWl$D(p8DloPbWP3%6Kk?EmM&uApCozU^>!Nx{+^pPQ z+^m*Zti~BzpP%k+JyLypk%!i5I;d)7yCRjr36)MPv__kcVrMhYEssN+(>j+jP?)na zuGEeIV`@u~_5KCR)K32jVm3o=cet3~$faFRQdvL&#b%g6fAitQveAAY=l znBI<2{Xx%`ueNqN;yzGYgN)|^&>KJEd6>1^?W`lTUJVvK*V1~>@T8Dtb`ywhc)Rw zN-rwP3y8r}K#zX8f1KixMgJrO-aq;KciPs!*N6@UCv(*MS9gorNo)OPNi@^i1bQMW z83IboJ`EP90hlQAzRk6(RIzgq4jCOPkTI!at8DUDJkXq@HkHeWS)q`C_Tqs((1*X= zKTSO;CL$dc*M%DB{ocXU7~4-eRN~Un!cX+)giy1}CQ&XivV+L^V^Ge(U5PkuG_N=}?1=NprMF&4$y}LZBs*wr!%LfeEA6j!4$M4jk^@fcxoq{ zjlK6v)s@Q>pdY2?vTi02*r{rq@=RxqJy*J0f@gr936L>yA>*hD(-Ksnc0vVHIIXS* zbQP{#L}y&@sdmj_1HFx_20cFhI2U84z~$fl6gWSFDBQZ*%pNd$-6HT*G5cG)&FPI(=2LSQlL|mwfv6gF5x@>dxTslr z=0L`z#b<}|x%1c%6e-7A2q;U*%`IcjD&fnpIWU33e1YD=u8NVB(N7U?>;*PGdXN`M>^h|?{NiRAfxqgKK8#vbj zlUX_>HA=x$v(QKgK702F84fV9*X0k|k)C;wg+zcG*6SjnDi|v;2&@}uxzbkNGN>t> zSEg;by3#e(Zj+1mla4^vfJ_$c6S5Se0s-xLa(`s?jBZ`KHkmYQi~v#AYlcQpKH^%9 a>Hh;ZA%(*0hPp)n0000F=4.2.6, <4.4.0", "databricks-sqlalchemy==1.0.5", ] +datafusion = ["flightsql-dbapi>=0.2.0, <0.3"] db2 = ["ibm-db-sa>0.3.8, <=0.4.4"] denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"] dremio = ["sqlalchemy-dremio>=1.2.1, <4"] diff --git a/superset/db_engine_specs/datafusion.py b/superset/db_engine_specs/datafusion.py new file mode 100644 index 00000000000..ec7a9ac4f64 --- /dev/null +++ b/superset/db_engine_specs/datafusion.py @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from typing import Any + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs import BaseEngineSpec +from superset.db_engine_specs.base import DatabaseCategory + + +class DataFusionEngineSpec(BaseEngineSpec): + engine_name = "Apache DataFusion" + engine = "datafusion" + drivers = { + "flightsql": "Arrow Flight SQL protocol for DataFusion", + } + default_driver = "flightsql" + sqlalchemy_uri_placeholder = "datafusion://host:port" + supports_file_upload = False + + metadata = { + "description": "DataFusion is a highly performant query engine", + "logo": "datafusion.png", + "homepage_url": "https://datafusion.apache.org/", + "categories": [ + DatabaseCategory.QUERY_ENGINES, + DatabaseCategory.OPEN_SOURCE, + DatabaseCategory.APACHE_PROJECTS, + ], + "pypi_packages": ["flightsql-dbapi"], + "connection_string": "datafusion://host:port", + "drivers": [ + { + "name": "Arrow Flight SQL (Recommended)", + "pypi_package": "flightsql-dbapi", + "connection_string": "datafusion://host:port", + "is_recommended": True, + } + ], + } + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DATE_TRUNC('second', {col})", + TimeGrain.FIVE_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '5 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 5)", # noqa: E501 + TimeGrain.THIRTY_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '30 seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 30)", # noqa: E501 + TimeGrain.MINUTE: "DATE_TRUNC('minute', {col})", + TimeGrain.FIVE_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '5 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 5)", # noqa: E501 + TimeGrain.TEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '10 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 10)", # noqa: E501 + TimeGrain.FIFTEEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '15 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 15)", # noqa: E501 + TimeGrain.THIRTY_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '30 minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 30)", # noqa: E501 + TimeGrain.HOUR: "DATE_TRUNC('hour', {col})", + TimeGrain.DAY: "DATE_TRUNC('day', {col})", + TimeGrain.WEEK: "DATE_TRUNC('week', {col})", + TimeGrain.MONTH: "DATE_TRUNC('month', {col})", + TimeGrain.QUARTER: "DATE_TRUNC('quarter', {col})", + TimeGrain.YEAR: "DATE_TRUNC('year', {col})", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "from_unixtime({col})" + + @classmethod + def convert_dttm( + cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None + ) -> str | None: + sqla_type = cls.get_sqla_column_type(target_type) + + if isinstance(sqla_type, types.Date): + return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" + if isinstance(sqla_type, types.DateTime): + dttm_formatted = dttm.isoformat(sep=" ", timespec="milliseconds") + return f"""TO_TIMESTAMP('{dttm_formatted}', 'YYYY-MM-DD HH24:MI:SS.FFF')""" + return None diff --git a/superset_text.yml b/superset_text.yml index 3b3a346b54f..aeb84d631b7 100644 --- a/superset_text.yml +++ b/superset_text.yml @@ -19,6 +19,7 @@ # To set the images of your preferred database, you may create a mapping here with engine and locations of the relevant images. The image can be hosted locally inside your static/file directory or online (e.g. S3). # DB_IMAGES: +# datafusion: "path/to/image/datafusion.jpg" # postgresql: "path/to/image/postgres.jpg" # bigquery: "path/to/s3bucket/bigquery.jpg" # snowflake: "path/to/image/snowflake.jpg" diff --git a/tests/unit_tests/db_engine_specs/test_datafusion.py b/tests/unit_tests/db_engine_specs/test_datafusion.py new file mode 100644 index 00000000000..0595f885cfd --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_datafusion.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime + +import pytest + +from superset.db_engine_specs.datafusion import DataFusionEngineSpec +from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm +from tests.unit_tests.fixtures.common import dttm # noqa: F401 + + +@pytest.mark.parametrize( + "target_type,expected_result", + [ + ("Date", "TO_DATE('2019-01-02', 'YYYY-MM-DD')"), + ( + "DateTime", + "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')", + ), + ( + "TimeStamp", + "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD HH24:MI:SS.FFF')", + ), + ("UnknownType", None), + ], +) +def test_convert_dttm( + target_type: str, + expected_result: str | None, + dttm: datetime, # noqa: F811 +) -> None: + assert_convert_dttm(DataFusionEngineSpec, target_type, expected_result, dttm) + + +def test_epoch_to_dttm() -> None: + assert DataFusionEngineSpec.epoch_to_dttm().format(col="ts") == "from_unixtime(ts)" + + +@pytest.mark.parametrize( + "time_grain,expected", + [ + (None, "ts"), + ("PT1S", "DATE_TRUNC('second', ts)"), + ( + "PT5S", + "DATE_TRUNC('minute', ts) + INTERVAL '5 seconds' * " + "FLOOR(EXTRACT(SECOND FROM ts) / 5)", + ), + ( + "PT30S", + "DATE_TRUNC('minute', ts) + INTERVAL '30 seconds' * " + "FLOOR(EXTRACT(SECOND FROM ts) / 30)", + ), + ("PT1M", "DATE_TRUNC('minute', ts)"), + ( + "PT5M", + "DATE_TRUNC('hour', ts) + INTERVAL '5 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 5)", + ), + ( + "PT10M", + "DATE_TRUNC('hour', ts) + INTERVAL '10 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 10)", + ), + ( + "PT15M", + "DATE_TRUNC('hour', ts) + INTERVAL '15 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 15)", + ), + ( + "PT30M", + "DATE_TRUNC('hour', ts) + INTERVAL '30 minutes' * " + "FLOOR(EXTRACT(MINUTE FROM ts) / 30)", + ), + ("PT1H", "DATE_TRUNC('hour', ts)"), + ("P1D", "DATE_TRUNC('day', ts)"), + ("P1W", "DATE_TRUNC('week', ts)"), + ("P1M", "DATE_TRUNC('month', ts)"), + ("P3M", "DATE_TRUNC('quarter', ts)"), + ("P1Y", "DATE_TRUNC('year', ts)"), + ], +) +def test_time_grain_expressions(time_grain: str | None, expected: str) -> None: + assert ( + DataFusionEngineSpec._time_grain_expressions[time_grain].format(col="ts") + == expected + )