mirror of
https://github.com/apache/superset.git
synced 2026-04-28 20:44:24 +00:00
Compare commits
2 Commits
docs/testi
...
fix-mysql-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c88f2d61d | ||
|
|
66f6a6ce94 |
@@ -77,6 +77,18 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
|
|||||||
supports_multivalues_insert = True
|
supports_multivalues_insert = True
|
||||||
|
|
||||||
column_type_mappings = (
|
column_type_mappings = (
|
||||||
|
# Boolean types - MySQL uses TINYINT(1) for BOOLEAN
|
||||||
|
(
|
||||||
|
re.compile(r"^tinyint\(1\)", re.IGNORECASE),
|
||||||
|
TINYINT(),
|
||||||
|
GenericDataType.BOOLEAN,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
re.compile(r"^bool(ean)?", re.IGNORECASE),
|
||||||
|
TINYINT(),
|
||||||
|
GenericDataType.BOOLEAN,
|
||||||
|
),
|
||||||
|
# Numeric types
|
||||||
(
|
(
|
||||||
re.compile(r"^int.*", re.IGNORECASE),
|
re.compile(r"^int.*", re.IGNORECASE),
|
||||||
INTEGER(),
|
INTEGER(),
|
||||||
@@ -260,6 +272,67 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
|
|||||||
def epoch_to_dttm(cls) -> str:
|
def epoch_to_dttm(cls) -> str:
|
||||||
return "from_unixtime({col})"
|
return "from_unixtime({col})"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _is_boolean_column(cls, col_desc: tuple[Any, ...]) -> bool:
|
||||||
|
"""Check if a cursor column description represents a boolean column."""
|
||||||
|
type_code = col_desc[1] if len(col_desc) > 1 else None
|
||||||
|
display_size = col_desc[2] if len(col_desc) > 2 else None
|
||||||
|
|
||||||
|
# Only process FIELD_TYPE.TINY (type_code 1)
|
||||||
|
if type_code != 1:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Explicit width 1 indicates TINYINT(1)/BOOLEAN
|
||||||
|
if display_size == 1:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Check SQLAlchemy type string (some drivers provide it at index 4)
|
||||||
|
if len(col_desc) > 4 and isinstance(col_desc[4], str):
|
||||||
|
sqla_type_str = col_desc[4].lower()
|
||||||
|
return any(marker in sqla_type_str for marker in ["bool", "tinyint(1)"])
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fetch_data(cls, cursor: Any, limit: int | None = None) -> list[tuple[Any, ...]]:
|
||||||
|
"""
|
||||||
|
Fetch data from cursor, converting MySQL TINYINT(1) values to Python booleans.
|
||||||
|
|
||||||
|
MySQL stores BOOLEAN as TINYINT(1), but returns 0/1 integers instead of
|
||||||
|
True/False. This method detects TINYINT(1) columns using multiple reliable
|
||||||
|
markers and converts their values to proper Python booleans.
|
||||||
|
"""
|
||||||
|
data = super().fetch_data(cursor, limit)
|
||||||
|
if not cursor.description:
|
||||||
|
return data
|
||||||
|
|
||||||
|
# Find TINYINT(1) columns
|
||||||
|
bool_column_indices = [
|
||||||
|
i
|
||||||
|
for i, col_desc in enumerate(cursor.description)
|
||||||
|
if cls._is_boolean_column(col_desc)
|
||||||
|
]
|
||||||
|
|
||||||
|
# Convert 0/1 to True/False for boolean columns
|
||||||
|
if bool_column_indices:
|
||||||
|
converted_data = []
|
||||||
|
for row in data:
|
||||||
|
new_row = list(row)
|
||||||
|
for col_idx in bool_column_indices:
|
||||||
|
if new_row[col_idx] is not None:
|
||||||
|
# Normalize different value types before boolean conversion
|
||||||
|
# bool("0") returns True, but we need False for MySQL boolean
|
||||||
|
value = new_row[col_idx]
|
||||||
|
if isinstance(value, (str, bytes)):
|
||||||
|
value = int(value)
|
||||||
|
elif isinstance(value, Decimal):
|
||||||
|
value = int(value)
|
||||||
|
new_row[col_idx] = bool(value)
|
||||||
|
converted_data.append(tuple(new_row))
|
||||||
|
return converted_data
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _extract_error_message(cls, ex: Exception) -> str:
|
def _extract_error_message(cls, ex: Exception) -> str:
|
||||||
"""Extract error message for queries"""
|
"""Extract error message for queries"""
|
||||||
|
|||||||
@@ -47,8 +47,16 @@ from tests.unit_tests.fixtures.common import dttm # noqa: F401
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"native_type,sqla_type,attrs,generic_type,is_dttm",
|
"native_type,sqla_type,attrs,generic_type,is_dttm",
|
||||||
[
|
[
|
||||||
# Numeric
|
# Boolean types - MySQL uses TINYINT(1) for BOOLEAN
|
||||||
|
("TINYINT(1)", TINYINT, None, GenericDataType.BOOLEAN, False),
|
||||||
|
("tinyint(1)", TINYINT, None, GenericDataType.BOOLEAN, False),
|
||||||
|
("BOOLEAN", TINYINT, None, GenericDataType.BOOLEAN, False),
|
||||||
|
("BOOL", TINYINT, None, GenericDataType.BOOLEAN, False),
|
||||||
|
# Numeric (ensure TINYINT without (1) remains numeric)
|
||||||
("TINYINT", TINYINT, None, GenericDataType.NUMERIC, False),
|
("TINYINT", TINYINT, None, GenericDataType.NUMERIC, False),
|
||||||
|
("TINYINT(2)", TINYINT, None, GenericDataType.NUMERIC, False),
|
||||||
|
("TINYINT(4)", TINYINT, None, GenericDataType.NUMERIC, False),
|
||||||
|
("TINYINT UNSIGNED", TINYINT, None, GenericDataType.NUMERIC, False),
|
||||||
("SMALLINT", types.SmallInteger, None, GenericDataType.NUMERIC, False),
|
("SMALLINT", types.SmallInteger, None, GenericDataType.NUMERIC, False),
|
||||||
("MEDIUMINT", MEDIUMINT, None, GenericDataType.NUMERIC, False),
|
("MEDIUMINT", MEDIUMINT, None, GenericDataType.NUMERIC, False),
|
||||||
("INT", INTEGER, None, GenericDataType.NUMERIC, False),
|
("INT", INTEGER, None, GenericDataType.NUMERIC, False),
|
||||||
@@ -77,9 +85,11 @@ def test_get_column_spec(
|
|||||||
generic_type: GenericDataType,
|
generic_type: GenericDataType,
|
||||||
is_dttm: bool,
|
is_dttm: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
assert_column_spec(spec, native_type, sqla_type, attrs, generic_type, is_dttm)
|
assert_column_spec(
|
||||||
|
MySQLEngineSpec, native_type, sqla_type, attrs, generic_type, is_dttm
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -98,9 +108,9 @@ def test_convert_dttm(
|
|||||||
expected_result: Optional[str],
|
expected_result: Optional[str],
|
||||||
dttm: datetime, # noqa: F811
|
dttm: datetime, # noqa: F811
|
||||||
) -> None:
|
) -> None:
|
||||||
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
assert_convert_dttm(spec, target_type, expected_result, dttm)
|
assert_convert_dttm(MySQLEngineSpec, target_type, expected_result, dttm)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -255,10 +265,153 @@ def test_column_type_mutator(
|
|||||||
description: list[Any],
|
description: list[Any],
|
||||||
expected_result: list[tuple[Any, ...]],
|
expected_result: list[tuple[Any, ...]],
|
||||||
):
|
):
|
||||||
from superset.db_engine_specs.mysql import MySQLEngineSpec as spec # noqa: N813
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
mock_cursor = Mock()
|
mock_cursor = Mock()
|
||||||
mock_cursor.fetchall.return_value = data
|
mock_cursor.fetchall.return_value = data
|
||||||
mock_cursor.description = description
|
mock_cursor.description = description
|
||||||
|
|
||||||
assert spec.fetch_data(mock_cursor) == expected_result
|
assert MySQLEngineSpec.fetch_data(mock_cursor) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_integers() -> None:
|
||||||
|
"""Test fetch_data converts integer 0/1 to boolean False/True."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [(1, "admin"), (0, "user")]
|
||||||
|
# TINYINT(1) column: type_code=1 (FIELD_TYPE.TINY), display_size=1
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_active", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
("role", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, "admin"), (False, "user")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_strings() -> None:
|
||||||
|
"""Test fetch_data converts string "0"/"1" to boolean False/True."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [("1", "admin"), ("0", "user")]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_active", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
("role", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, "admin"), (False, "user")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_bytes() -> None:
|
||||||
|
"""Test fetch_data converts bytes b"0"/b"1" to boolean False/True."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [(b"1", "admin"), (b"0", "user")]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_active", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
("role", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, "admin"), (False, "user")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_decimals() -> None:
|
||||||
|
"""Test fetch_data converts Decimal 0/1 to boolean False/True."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [
|
||||||
|
(Decimal("1"), "admin"),
|
||||||
|
(Decimal("0"), "user"),
|
||||||
|
]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_active", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
("role", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, "admin"), (False, "user")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_with_nulls() -> None:
|
||||||
|
"""Test fetch_data handles NULL values correctly in boolean columns."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [(1, "admin"), (None, "user"), (0, "guest")]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_active", 1, 1, 4, 3, 0, True), # TINYINT(1) with nulls - should convert
|
||||||
|
("role", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, "admin"), (None, "user"), (False, "guest")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_mixed_columns() -> None:
|
||||||
|
"""Test fetch_data with boolean and non-boolean columns together."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [(1, 50, 0), (0, 100, 1)]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("is_admin", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
("count", 3, 11, 4, 10, 0, False), # INT - should not convert
|
||||||
|
("is_active", 1, 1, 4, 3, 0, False), # TINYINT(1) - should convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(True, 50, False), (False, 100, True)]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_no_boolean_columns() -> None:
|
||||||
|
"""Test fetch_data passes through data when no boolean columns present."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
mock_cursor.fetchall.return_value = [(100, "test"), (200, "data")]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("count", 3, 11, 4, 10, 0, False), # INT - should not convert
|
||||||
|
("name", 254, 255, 0, 0, 0, False), # VARCHAR - should not convert
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [(100, "test"), (200, "data")]
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_data_boolean_mixed_driver_types() -> None:
|
||||||
|
"""Test fetch_data with different driver return types in same dataset."""
|
||||||
|
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||||
|
|
||||||
|
mock_cursor = Mock()
|
||||||
|
# Mix of integers, strings, bytes, decimals for boolean values
|
||||||
|
mock_cursor.fetchall.return_value = [
|
||||||
|
(1, "0", b"1"), # True, False, True
|
||||||
|
(0, "1", b"0"), # False, True, False
|
||||||
|
(Decimal(1), None, 0), # True, None, False
|
||||||
|
]
|
||||||
|
mock_cursor.description = [
|
||||||
|
("bool_int", 1, 1, 4, 3, 0, False), # TINYINT(1) - integers
|
||||||
|
("bool_str", 1, 1, 4, 3, 0, False), # TINYINT(1) - strings
|
||||||
|
("bool_bytes", 1, 1, 4, 3, 0, False), # TINYINT(1) - bytes
|
||||||
|
]
|
||||||
|
|
||||||
|
result = MySQLEngineSpec.fetch_data(mock_cursor)
|
||||||
|
expected = [
|
||||||
|
(True, False, True),
|
||||||
|
(False, True, False),
|
||||||
|
(True, None, False),
|
||||||
|
]
|
||||||
|
assert result == expected
|
||||||
|
|||||||
Reference in New Issue
Block a user