QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-08-12 11:11:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add option for hash algorithms (#35621)

Browse Source 
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
This commit is contained in:
Daniel Vaz Gaspar
2025-12-09 16:59:07 +00:00
committed by GitHub
co-authored by Ville Brofeldt
parent 8d7c83419c
commit bb22eb1ca8
36 changed files with 1032 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration_tests/db_engine_specs/bigquery_tests.py
+4 -3
View File
@@ -47,11 +47,12 @@ class TestBigQueryDbEngineSpec(SupersetTestCase):
"""
DB Eng Specs (bigquery): Test column label
"""
# Expected labels with SHA-256 hash suffix (first 5 chars prefixed with _)
test_cases = {
"Col": "Col",
"SUM(x)": "SUM_x__5f110",
"SUM[x]": "SUM_x__7ebe1",
"12345_col": "_12345_col_8d390",
"SUM(x)": "SUM_x__b681e",
"SUM[x]": "SUM_x__ceaf6",
"12345_col": "_12345_col_b1415",
}
for original, expected in test_cases.items():
actual = BigQueryEngineSpec.make_label_compatible(column(original).name)
tests/integration_tests/sqla_models_tests.py
+2 -1
View File
@@ -520,7 +520,8 @@ class TestDatabaseModel(SupersetTestCase):
sqlaq = table.get_sqla_query(**query_obj)
assert sqlaq.labels_expected == ["user", "COUNT_DISTINCT(user)"]
sql = table.database.compile_sqla_query(sqlaq.sqla_query)
assert "COUNT_DISTINCT_user__00db1" in sql
# SHA-256 hash of "COUNT_DISTINCT(user)" starts with "01c94"
assert "COUNT_DISTINCT_user__01c94" in sql
db.session.delete(table)
db.session.delete(database)
db.session.commit()
tests/integration_tests/thumbnails_tests.py
+2 -1
View File
@@ -191,7 +191,8 @@ class TestWebDriverSelenium(SupersetTestCase):
class TestThumbnails(SupersetTestCase):
mock_image = b"bytes mock image"
digest_return_value = "foo_bar"
digest_hash = "5c7d96a3dd7a87850a2ef34087565a6e"
# SHA-256 hash of "foo_bar" (default HASH_ALGORITHM is sha256)
digest_hash = "4928cae8b37b3d1113f5e01e60c967df6c2b9e826dc7d91488d23a62fec715ba"
def _get_id_and_thumbnail_url(self, url: str) -> tuple[int, str]:
rv = self.client.get(url)
tests/integration_tests/utils/core_tests.py
+17 -7
View File
@@ -19,7 +19,7 @@ import pytest
from superset.utils.core import form_data_to_adhoc, simple_filter_to_adhoc
def test_simple_filter_to_adhoc_generates_deterministic_values():
def test_simple_filter_to_adhoc_generates_deterministic_values(app_context):
input_1 = {
"op": "IS NOT NULL",
"col": "LATITUDE",
@@ -30,13 +30,16 @@ def test_simple_filter_to_adhoc_generates_deterministic_values():
# The result is the same when given the same input
assert simple_filter_to_adhoc(input_1) == simple_filter_to_adhoc(input_1)
# SHA-256 filterOptionName hash with default HASH_ALGORITHM
assert simple_filter_to_adhoc(input_1) == {
"clause": "WHERE",
"expressionType": "SIMPLE",
"comparator": "",
"operator": "IS NOT NULL",
"subject": "LATITUDE",
"filterOptionName": "6ac89d498115da22396f80a765cffc70",
"filterOptionName": (
"84ffe4dba1764c30568e19d4dbbf64717fbc514fad1a8a995debfc72b344aa76"
),
}
# The result is different when given different input
@@ -47,22 +50,27 @@ def test_simple_filter_to_adhoc_generates_deterministic_values():
"comparator": "",
"operator": "IS NOT NULL",
"subject": "LONGITUDE",
"filterOptionName": "9c984bd3714883ca859948354ce26ab9",
"filterOptionName": (
"c5a54054b987350b5594ee73772fbe71e9651a475bfcb7ae740e0799f12c8ff7"
),
}
def test_form_data_to_adhoc_generates_deterministic_values():
def test_form_data_to_adhoc_generates_deterministic_values(app_context):
form_data = {"where": "1 = 1", "having": "count(*) > 1"}
# The result is the same when given the same input
assert form_data_to_adhoc(form_data, "where") == form_data_to_adhoc(
form_data, "where"
)
# SHA-256 filterOptionName hash with default HASH_ALGORITHM
assert form_data_to_adhoc(form_data, "where") == {
"clause": "WHERE",
"expressionType": "SQL",
"sqlExpression": "1 = 1",
"filterOptionName": "99fe79985afbddea4492626dc6a87b74",
"filterOptionName": (
"11f7ef40818a0d614cc9a989d5d75ee969b5b3724e973dbf0194e3a339aa0544"
),
}
# The result is different when given different input
@@ -73,11 +81,13 @@ def test_form_data_to_adhoc_generates_deterministic_values():
"clause": "HAVING",
"expressionType": "SQL",
"sqlExpression": "count(*) > 1",
"filterOptionName": "1da11f6b709c3190daeabb84f77fc8c2",
"filterOptionName": (
"8768cb92fa8a8629695dfe3a4010daefc5d7586934d1aa775f22fb03b46b5dcb"
),
}
def test_form_data_to_adhoc_incorrect_clause_type():
def test_form_data_to_adhoc_incorrect_clause_type(app_context):
form_data = {"where": "1 = 1", "having": "count(*) > 1"}
with pytest.raises(ValueError): # noqa: PT011
tests/integration_tests/utils/hashing_tests.py
+155 -51
View File
@@ -17,80 +17,184 @@
import datetime
import math
from typing import Any
from unittest.mock import patch
import pytest # noqa: F401
from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str
from superset.utils.hashing import hash_from_dict, hash_from_str
def test_basic_md5_sha():
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
"""Test basic hashing with MD5 (legacy mode)."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="md5"):
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
serialized_obj = (
'{"company": "Gobias Industries", "price_in_cents": 4000, "product": "Coffee"}'
)
serialized_obj = '{"company": "Gobias Industries", "price_in_cents": 4000, "product": "Coffee"}' # noqa: E501
assert md5_sha_from_str(serialized_obj) == md5_sha_from_dict(obj)
assert md5_sha_from_str(serialized_obj) == "35f22273cd6a6798b04f8ddef51135e3"
assert hash_from_str(serialized_obj) == hash_from_dict(obj)
assert hash_from_str(serialized_obj) == "35f22273cd6a6798b04f8ddef51135e3"
def test_basic_sha256():
"""Test basic hashing with SHA-256 (FedRAMP compliant mode)."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="sha256"):
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
serialized_obj = '{"company": "Gobias Industries", "price_in_cents": 4000, "product": "Coffee"}' # noqa: E501
assert hash_from_str(serialized_obj) == hash_from_dict(obj)
# SHA-256 hash of the serialized object
assert (
hash_from_str(serialized_obj)
== "77bc5927f828903888572ab91c4f3114b36609ca5fb92039bef380d622cef596"
)
def test_sort_order_md5_sha():
obj_1 = {
"product": "Coffee",
"price_in_cents": 4000,
"company": "Gobias Industries",
}
"""Test dictionary key order independence with MD5."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="md5"):
obj_1 = {
"product": "Coffee",
"price_in_cents": 4000,
"company": "Gobias Industries",
}
obj_2 = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
obj_2 = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
assert md5_sha_from_dict(obj_1) == md5_sha_from_dict(obj_2)
assert md5_sha_from_dict(obj_1) == "35f22273cd6a6798b04f8ddef51135e3"
assert hash_from_dict(obj_1) == hash_from_dict(obj_2)
assert hash_from_dict(obj_1) == "35f22273cd6a6798b04f8ddef51135e3"
def test_sort_order_sha256():
"""Test dictionary key order independence with SHA-256."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="sha256"):
obj_1 = {
"product": "Coffee",
"price_in_cents": 4000,
"company": "Gobias Industries",
}
obj_2 = {
"product": "Coffee",
"company": "Gobias Industries",
"price_in_cents": 4000,
}
assert hash_from_dict(obj_1) == hash_from_dict(obj_2)
assert (
hash_from_dict(obj_1)
== "77bc5927f828903888572ab91c4f3114b36609ca5fb92039bef380d622cef596"
)
def test_custom_default_md5_sha():
def custom_datetime_serializer(obj: Any):
if isinstance(obj, datetime.datetime):
return "<datetime>"
"""Test custom serializer with MD5."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="md5"):
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"datetime": datetime.datetime.now(),
}
def custom_datetime_serializer(obj: Any):
if isinstance(obj, datetime.datetime):
return "<datetime>"
serialized_obj = '{"company": "Gobias Industries", "datetime": "<datetime>", "product": "Coffee"}' # noqa: E501
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"datetime": datetime.datetime.now(),
}
assert md5_sha_from_str(serialized_obj) == md5_sha_from_dict(
obj, default=custom_datetime_serializer
)
assert md5_sha_from_str(serialized_obj) == "dc280121213aabcaeb8087aef268fd0d"
serialized_obj = '{"company": "Gobias Industries", "datetime": "<datetime>", "product": "Coffee"}' # noqa: E501
assert hash_from_str(serialized_obj) == hash_from_dict(
obj, default=custom_datetime_serializer
)
assert hash_from_str(serialized_obj) == "dc280121213aabcaeb8087aef268fd0d"
def test_custom_default_sha256():
"""Test custom serializer with SHA-256."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="sha256"):
def custom_datetime_serializer(obj: Any):
if isinstance(obj, datetime.datetime):
return "<datetime>"
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"datetime": datetime.datetime.now(),
}
serialized_obj = '{"company": "Gobias Industries", "datetime": "<datetime>", "product": "Coffee"}' # noqa: E501
assert hash_from_str(serialized_obj) == hash_from_dict(
obj, default=custom_datetime_serializer
)
assert (
hash_from_str(serialized_obj)
== "417b57b6f3979bdd0937286f2dc872089fcd5fdb7daad1d3dbcaae1e34cc564e"
)
def test_ignore_nan_md5_sha():
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price": math.nan,
}
"""Test NaN handling with MD5."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="md5"):
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price": math.nan,
}
serialized_obj = (
'{"company": "Gobias Industries", "price": NaN, "product": "Coffee"}'
)
serialized_obj = (
'{"company": "Gobias Industries", "price": NaN, "product": "Coffee"}'
)
assert md5_sha_from_str(serialized_obj) == md5_sha_from_dict(obj)
assert md5_sha_from_str(serialized_obj) == "5d129d1dffebc0bacc734366476d586d"
assert hash_from_str(serialized_obj) == hash_from_dict(obj)
assert hash_from_str(serialized_obj) == "5d129d1dffebc0bacc734366476d586d"
serialized_obj = (
'{"company": "Gobias Industries", "price": null, "product": "Coffee"}'
)
serialized_obj = (
'{"company": "Gobias Industries", "price": null, "product": "Coffee"}'
)
assert md5_sha_from_str(serialized_obj) == md5_sha_from_dict(obj, ignore_nan=True)
assert md5_sha_from_str(serialized_obj) == "40e87d61f6add03816bccdeac5713b9f"
assert hash_from_str(serialized_obj) == hash_from_dict(obj, ignore_nan=True)
assert hash_from_str(serialized_obj) == "40e87d61f6add03816bccdeac5713b9f"
def test_ignore_nan_sha256():
"""Test NaN handling with SHA-256."""
with patch("superset.utils.hashing.get_hash_algorithm", return_value="sha256"):
obj = {
"product": "Coffee",
"company": "Gobias Industries",
"price": math.nan,
}
serialized_obj = (
'{"company": "Gobias Industries", "price": NaN, "product": "Coffee"}'
)
assert hash_from_str(serialized_obj) == hash_from_dict(obj)
assert (
hash_from_str(serialized_obj)
== "efff87146d137b2d0392eff94b74e7644c3a6b135b91563400029995b9236820"
)
serialized_obj = (
'{"company": "Gobias Industries", "price": null, "product": "Coffee"}'
)
assert hash_from_str(serialized_obj) == hash_from_dict(obj, ignore_nan=True)
assert (
hash_from_str(serialized_obj)
== "9b66e0af1cb74aa58c3ab08654c086ebfdada14b1e6312b4002edc854d99d24d"
)
tests/integration_tests/utils_tests.py
+18 -6
View File
@@ -60,7 +60,7 @@ from superset.utils.core import (
from superset.utils import json
from superset.utils.database import get_or_create_db
from superset.utils import schema
from superset.utils.hashing import md5_sha_from_str
from superset.utils.hashing import hash_from_str
from superset.views.utils import build_extra_filters, get_form_data # noqa: F401
from tests.integration_tests.base_tests import SupersetTestCase
from tests.integration_tests.constants import ADMIN_USERNAME
@@ -80,7 +80,10 @@ class TestUtils(SupersetTestCase):
{
"clause": "WHERE",
"expressionType": "SQL",
"filterOptionName": "46fb6d7891e23596e42ae38da94a57e0",
# SHA-256 hash with default HASH_ALGORITHM
"filterOptionName": (
"efcc050e11722b0bc338c0abc71a4270ce71df7a10294fcf8e8f03f5cb8978f3"
),
"sqlExpression": "a = 1",
}
]
@@ -96,7 +99,10 @@ class TestUtils(SupersetTestCase):
"clause": "WHERE",
"comparator": "someval",
"expressionType": "SIMPLE",
"filterOptionName": "135c7ee246666b840a3d7a9c3a30cf38",
# SHA-256 hash with default HASH_ALGORITHM
"filterOptionName": (
"d72b098cd87dc5040410c322373562ca65d1a736e1e53e9cae39254394b42a44"
),
"operator": "in",
"subject": "a",
}
@@ -112,7 +118,10 @@ class TestUtils(SupersetTestCase):
{
"clause": "WHERE",
"expressionType": "SQL",
"filterOptionName": "46fb6d7891e23596e42ae38da94a57e0",
# SHA-256 hash with default HASH_ALGORITHM
"filterOptionName": (
"efcc050e11722b0bc338c0abc71a4270ce71df7a10294fcf8e8f03f5cb8978f3"
),
"sqlExpression": "a = 1",
}
]
@@ -127,7 +136,10 @@ class TestUtils(SupersetTestCase):
{
"clause": "HAVING",
"expressionType": "SQL",
"filterOptionName": "683f1c26466ab912f75a00842e0f2f7b",
# SHA-256 hash with default HASH_ALGORITHM
"filterOptionName": (
"63a84e72e4dac2bb08de866699d9c4f8ccc3640f6c3c0b734c75b937fac54bd6"
),
"sqlExpression": "COUNT(1) = 1",
}
]
@@ -266,7 +278,7 @@ class TestUtils(SupersetTestCase):
def test_ssl_certificate_file_creation(self):
path = create_ssl_cert_file(ssl_certificate)
expected_filename = md5_sha_from_str(ssl_certificate)
expected_filename = hash_from_str(ssl_certificate)
assert expected_filename in path
assert os.path.exists(path)
tests/integration_tests/viz_tests.py
+13 -4
View File
@@ -1145,12 +1145,15 @@ class TestBaseDeckGLViz(SupersetTestCase):
}
datasource = self.get_datasource_mock()
# SHA-256 filterOptionName hashes with default HASH_ALGORITHM
expected_results = {
"latlong_key": [
{
"clause": "WHERE",
"expressionType": "SIMPLE",
"filterOptionName": "c7f171cf3204bcbf456acfeac5cd9afd",
"filterOptionName": (
"980dd3068274177120307d9182ea8e8ee1b7824d34fbc21c529441f5d3279f7f"
),
"comparator": "",
"operator": "IS NOT NULL",
"subject": "lat",
@@ -1158,7 +1161,9 @@ class TestBaseDeckGLViz(SupersetTestCase):
{
"clause": "WHERE",
"expressionType": "SIMPLE",
"filterOptionName": "52634073fbb8ae0a3aa59ad48abac55e",
"filterOptionName": (
"e368c259da27e5ec6a854772d9bff2c2af8dd5762352cef4ff6afc5bd8b6b9ea"
),
"comparator": "",
"operator": "IS NOT NULL",
"subject": "lon",
@@ -1168,7 +1173,9 @@ class TestBaseDeckGLViz(SupersetTestCase):
{
"clause": "WHERE",
"expressionType": "SIMPLE",
"filterOptionName": "cae5c925c140593743da08499e6fb207",
"filterOptionName": (
"6ea33b70ab781033af421240019d3e3ad782928a3ad2999538f1f4b2a52305e2"
),
"comparator": "",
"operator": "IS NOT NULL",
"subject": "lonlat",
@@ -1178,7 +1185,9 @@ class TestBaseDeckGLViz(SupersetTestCase):
{
"clause": "WHERE",
"expressionType": "SIMPLE",
"filterOptionName": "d84f55222d8e414e888fa5f990b341d2",
"filterOptionName": (
"48bbd94cd6afb1885d8550e2928bc01a2d3bc7d1f4f1d0929b10d6f4021b7f14"
),
"comparator": "",
"operator": "IS NOT NULL",
"subject": "geo",