QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(SIP-95): new endpoint for table metadata (#28122)

Browse Source
This commit is contained in:
Beto Dealmeida
2024-04-25 12:23:49 -04:00
committed by GitHub
parent 52f8734662
commit 6cf681df68
71 changed files with 1048 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/unit_tests/dao/dataset_test.py
View File

@@ -18,6 +18,7 @@
from sqlalchemy.orm.session import Session
from superset.daos.dataset import DatasetDAO
from superset.sql_parse import Table
def test_validate_update_uniqueness(session: Session) -> None:
@@ -54,9 +55,8 @@ def test_validate_update_uniqueness(session: Session) -> None:
assert (
DatasetDAO.validate_update_uniqueness(
database_id=database.id,
schema=dataset1.schema,
table=Table(dataset1.table_name, dataset1.schema),
dataset_id=dataset1.id,
name=dataset1.table_name,
)
is True
)
@@ -65,9 +65,8 @@ def test_validate_update_uniqueness(session: Session) -> None:
assert (
DatasetDAO.validate_update_uniqueness(
database_id=database.id,
schema=dataset2.schema,
table=Table(dataset1.table_name, dataset2.schema),
dataset_id=dataset1.id,
name=dataset1.table_name,
)
is False
)
@@ -76,9 +75,8 @@ def test_validate_update_uniqueness(session: Session) -> None:
assert (
DatasetDAO.validate_update_uniqueness(
database_id=database.id,
schema=None,
table=Table(dataset1.table_name),
dataset_id=dataset1.id,
name=dataset1.table_name,
)
is True
)

164
tests/unit_tests/databases/api_test.py
View File

@@ -1415,6 +1415,170 @@ def test_excel_upload_file_extension_invalid(
assert response.json == {"message": {"file": ["File extension is not allowed."]}}
def test_table_metadata_happy_path(
mocker: MockFixture,
client: Any,
full_api_access: None,
) -> None:
"""
Test the `table_metadata` endpoint.
"""
database = mocker.MagicMock()
database.db_engine_spec.get_table_metadata.return_value = {"hello": "world"}
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
mocker.patch("superset.databases.api.security_manager.raise_for_access")
response = client.get("/api/v1/database/1/table_metadata/?name=t")
assert response.json == {"hello": "world"}
database.db_engine_spec.get_table_metadata.assert_called_with(
database,
Table("t"),
)
response = client.get("/api/v1/database/1/table_metadata/?name=t&schema=s")
database.db_engine_spec.get_table_metadata.assert_called_with(
database,
Table("t", "s"),
)
response = client.get("/api/v1/database/1/table_metadata/?name=t&catalog=c")
database.db_engine_spec.get_table_metadata.assert_called_with(
database,
Table("t", None, "c"),
)
response = client.get(
"/api/v1/database/1/table_metadata/?name=t&schema=s&catalog=c"
)
database.db_engine_spec.get_table_metadata.assert_called_with(
database,
Table("t", "s", "c"),
)
def test_table_metadata_no_table(
mocker: MockFixture,
client: Any,
full_api_access: None,
) -> None:
"""
Test the `table_metadata` endpoint when no table name is passed.
"""
database = mocker.MagicMock()
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
response = client.get("/api/v1/database/1/table_metadata/?schema=s&catalog=c")
assert response.status_code == 422
assert response.json == {
"errors": [
{
"message": "An error happened when validating the request",
"error_type": "INVALID_PAYLOAD_SCHEMA_ERROR",
"level": "error",
"extra": {
"messages": {"name": ["Missing data for required field."]},
"issue_codes": [
{
"code": 1020,
"message": "Issue 1020 - The submitted payload has the incorrect schema.",
}
],
},
}
]
}
def test_table_metadata_slashes(
mocker: MockFixture,
client: Any,
full_api_access: None,
) -> None:
"""
Test the `table_metadata` endpoint with names that have slashes.
"""
database = mocker.MagicMock()
database.db_engine_spec.get_table_metadata.return_value = {"hello": "world"}
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
mocker.patch("superset.databases.api.security_manager.raise_for_access")
client.get("/api/v1/database/1/table_metadata/?name=foo/bar")
database.db_engine_spec.get_table_metadata.assert_called_with(
database,
Table("foo/bar"),
)
def test_table_metadata_invalid_database(
mocker: MockFixture,
client: Any,
full_api_access: None,
) -> None:
"""
Test the `table_metadata` endpoint when the database is invalid.
"""
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=None)
response = client.get("/api/v1/database/1/table_metadata/?name=t")
assert response.status_code == 404
assert response.json == {
"errors": [
{
"message": "No such database",
"error_type": "DATABASE_NOT_FOUND_ERROR",
"level": "error",
"extra": {
"issue_codes": [
{
"code": 1011,
"message": "Issue 1011 - Superset encountered an unexpected error.",
},
{
"code": 1036,
"message": "Issue 1036 - The database was deleted.",
},
]
},
}
]
}
def test_table_metadata_unauthorized(
mocker: MockFixture,
client: Any,
full_api_access: None,
) -> None:
"""
Test the `table_metadata` endpoint when the user is unauthorized.
"""
database = mocker.MagicMock()
mocker.patch("superset.databases.api.DatabaseDAO.find_by_id", return_value=database)
mocker.patch(
"superset.databases.api.security_manager.raise_for_access",
side_effect=SupersetSecurityException(
SupersetError(
error_type=SupersetErrorType.TABLE_SECURITY_ACCESS_ERROR,
message="You don't have access to the table",
level=ErrorLevel.ERROR,
)
),
)
response = client.get("/api/v1/database/1/table_metadata/?name=t")
assert response.status_code == 404
assert response.json == {
"errors": [
{
"message": "No such table",
"error_type": "TABLE_NOT_FOUND_ERROR",
"level": "error",
"extra": None,
}
]
}
def test_table_extra_metadata_happy_path(
mocker: MockFixture,
client: Any,

6
tests/unit_tests/db_engine_specs/test_base.py
View File

@@ -232,9 +232,8 @@ def test_select_star(mocker: MockFixture) -> None:
sql = BaseEngineSpec.select_star(
database=database,
table_name="my_table",
table=Table("my_table"),
engine=engine,
schema=None,
limit=100,
show_cols=True,
indent=True,
@@ -252,9 +251,8 @@ OFFSET ?"""
sql = NoLimitDBEngineSpec.select_star(
database=database,
table_name="my_table",
table=Table("my_table"),
engine=engine,
schema=None,
limit=100,
show_cols=True,
indent=True,

4
tests/unit_tests/db_engine_specs/test_bigquery.py
View File

@@ -27,6 +27,7 @@ from sqlalchemy import select
from sqlalchemy.sql import sqltypes
from sqlalchemy_bigquery import BigQueryDialect
from superset.sql_parse import Table
from superset.superset_typing import ResultSetColumnType
from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm
from tests.unit_tests.fixtures.common import dttm # noqa: F401
@@ -156,9 +157,8 @@ def test_select_star(mocker: MockFixture) -> None:
sql = BigQueryEngineSpec.select_star(
database=database,
table_name="my_table",
table=Table("my_table"),
engine=engine,
schema=None,
limit=100,
show_cols=True,
indent=True,

7
tests/unit_tests/db_engine_specs/test_db2.py
View File

@@ -18,6 +18,8 @@
import pytest # noqa: F401
from pytest_mock import MockerFixture
from superset.sql_parse import Table
def test_epoch_to_dttm() -> None:
"""
@@ -43,7 +45,7 @@ def test_get_table_comment(mocker: MockerFixture):
}
assert (
Db2EngineSpec.get_table_comment(mock_inspector, "my_table", "my_schema")
Db2EngineSpec.get_table_comment(mock_inspector, Table("my_table", "my_schema"))
== "This is a table comment"
)
@@ -59,7 +61,8 @@ def test_get_table_comment_empty(mocker: MockerFixture):
mock_inspector.get_table_comment.return_value = {}
assert (
Db2EngineSpec.get_table_comment(mock_inspector, "my_table", "my_schema") is None # noqa: E711
Db2EngineSpec.get_table_comment(mock_inspector, Table("my_table", "my_schema"))
is None
)

6
tests/unit_tests/db_engine_specs/test_presto.py
View File

@@ -24,6 +24,7 @@ from pyhive.sqlalchemy_presto import PrestoDialect
from sqlalchemy import sql, text, types
from sqlalchemy.engine.url import make_url
from superset.sql_parse import Table
from superset.superset_typing import ResultSetColumnType
from superset.utils.core import GenericDataType
from tests.unit_tests.db_engine_specs.utils import (
@@ -143,7 +144,10 @@ def test_where_latest_partition(
expected = f"""SELECT * FROM table \nWHERE "partition_key" = {expected_value}"""
result = spec.where_latest_partition(
"table", mock.MagicMock(), mock.MagicMock(), query, columns
mock.MagicMock(),
Table("table"),
query,
columns,
)
assert result is not None
actual = result.compile(

16
tests/unit_tests/db_engine_specs/test_trino.py
View File

@@ -311,15 +311,15 @@ def test_convert_dttm(
assert_convert_dttm(TrinoEngineSpec, target_type, expected_result, dttm)
def test_get_extra_table_metadata() -> None:
def test_get_extra_table_metadata(mocker: MockerFixture) -> None:
from superset.db_engine_specs.trino import TrinoEngineSpec
db_mock = Mock()
db_mock = mocker.MagicMock()
db_mock.get_indexes = Mock(
return_value=[{"column_names": ["ds", "hour"], "name": "partition"}]
)
db_mock.get_extra = Mock(return_value={})
db_mock.has_view_by_name = Mock(return_value=None)
db_mock.has_view = Mock(return_value=None)
db_mock.get_df = Mock(return_value=pd.DataFrame({"ds": ["01-01-19"], "hour": [1]}))
result = TrinoEngineSpec.get_extra_table_metadata(
db_mock,
@@ -442,7 +442,7 @@ def test_get_columns(mocker: MockerFixture):
mock_inspector = mocker.MagicMock()
mock_inspector.get_columns.return_value = sqla_columns
actual = TrinoEngineSpec.get_columns(mock_inspector, "table", "schema")
actual = TrinoEngineSpec.get_columns(mock_inspector, Table("table", "schema"))
expected = [
ResultSetColumnType(
name="field1", column_name="field1", type=field1_type, is_dttm=False
@@ -475,7 +475,9 @@ def test_get_columns_expand_rows(mocker: MockerFixture):
mock_inspector.get_columns.return_value = sqla_columns
actual = TrinoEngineSpec.get_columns(
mock_inspector, "table", "schema", {"expand_rows": True}
mock_inspector,
Table("table", "schema"),
{"expand_rows": True},
)
expected = [
ResultSetColumnType(
@@ -538,7 +540,9 @@ def test_get_indexes_no_table():
side_effect=NoSuchTableError("The specified table does not exist.")
)
result = TrinoEngineSpec.get_indexes(
db_mock, inspector_mock, "test_table", "test_schema"
db_mock,
inspector_mock,
Table("test_table", "test_schema"),
)
assert result == []

9
tests/unit_tests/models/core_test.py
View File

@@ -18,7 +18,6 @@
# pylint: disable=import-outside-toplevel
import json
from datetime import datetime
from typing import Optional
import pytest
from pytest_mock import MockFixture
@@ -26,6 +25,7 @@ from sqlalchemy.engine.reflection import Inspector
from superset.connectors.sqla.models import SqlaTable, TableColumn
from superset.models.core import Database
from superset.sql_parse import Table
def test_get_metrics(mocker: MockFixture) -> None:
@@ -37,7 +37,7 @@ def test_get_metrics(mocker: MockFixture) -> None:
from superset.models.core import Database
database = Database(database_name="my_database", sqlalchemy_uri="sqlite://")
assert database.get_metrics("table") == [
assert database.get_metrics(Table("table")) == [
{
"expression": "COUNT(*)",
"metric_name": "count",
@@ -52,8 +52,7 @@ def test_get_metrics(mocker: MockFixture) -> None:
cls,
database: Database,
inspector: Inspector,
table_name: str,
schema: Optional[str],
table: Table,
) -> list[MetricType]:
return [
{
@@ -65,7 +64,7 @@ def test_get_metrics(mocker: MockFixture) -> None:
]
database.get_db_engine_spec = mocker.MagicMock(return_value=CustomSqliteEngineSpec)
assert database.get_metrics("table") == [
assert database.get_metrics(Table("table")) == [
{
"expression": "COUNT(DISTINCT user_id)",
"metric_name": "count_distinct_user_id",