Compare commits

...

1 Commits

Author SHA1 Message Date
Ville Brofeldt
a35c17cebf feat: break out superset-core package 2024-12-04 18:11:36 -08:00
36 changed files with 113 additions and 87 deletions

View File

@@ -95,6 +95,7 @@ dependencies = [
#https://github.com/tobymao/sqlglot/blob/main/CHANGELOG.md#v25250---2024-10-14
"sqlglot>=25.24.0,<25.25.0",
"sqlparse>=0.5.0",
"superset-core",
"tabulate>=0.8.9, <0.9",
"typing-extensions>=4, <5",
"waitress; sys_platform == 'win32'",

View File

@@ -17,6 +17,7 @@
# under the License.
#
-e file:.
-e file:superset-core/.
urllib3>=1.26.18
werkzeug>=3.0.1
numexpr>=2.9.0

View File

@@ -1,4 +1,4 @@
# SHA1:04f7e0860829f18926ea238354e6d4a6ab823d50
# SHA1:5db27c2c8daa8730583759637e8e8ae11ef40369
#
# This file is autogenerated by pip-compile-multi
# To update, run:
@@ -7,6 +7,8 @@
#
-e file:.
# via -r requirements/base.in
-e file:superset-core/.
# via -r requirements/base.in
alembic==1.14.0
# via flask-migrate
amqp==5.3.1

View File

@@ -6,10 +6,12 @@
# pip-compile-multi
#
-r base.txt
-e file:.
-e file:///Users/ville/apple/apache-superset
# via
# -r requirements/base.in
# -r /Users/ville/apple/apache-superset/requirements/base.in
# -r requirements/development.in
-e file:superset-core/.
# via -r /Users/ville/apple/apache-superset/requirements/base.in
astroid==3.1.0
# via pylint
build==1.2.1

View File

@@ -0,0 +1,12 @@
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "superset-core"
version = "0.1.0"
description = "Core stuff for Superset backend"
authors = [
{ name = "Apache Software Foundation", email = "dev@superset.apache.org" },
]
dependencies = []

View File

View File

@@ -0,0 +1,35 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from enum import IntEnum
class GenericDataType(IntEnum):
"""
Generic database column type that fits both frontend and backend.
"""
NUMERIC = 0
STRING = 1
TEMPORAL = 2
BOOLEAN = 3
# Mapping all the complex data types to STRING for now and leaving these as a
# reminder.
# ARRAY = 4
# JSON = 5
# MAP = 6
# ROW = 7

View File

@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class ChartDataResultFormat(StrEnum):

View File

@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class QueryStatus(StrEnum):

View File

@@ -20,6 +20,7 @@ import logging
from typing import Any, ClassVar, TYPE_CHECKING
import pandas as pd
from superset_core.charts.types import GenericDataType
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
from superset.common.query_context_processor import (
@@ -28,7 +29,6 @@ from superset.common.query_context_processor import (
)
from superset.common.query_object import QueryObject
from superset.models.slice import Slice
from superset.utils.core import GenericDataType
if TYPE_CHECKING:
from superset.connectors.sqla.models import BaseDatasource

View File

@@ -26,6 +26,7 @@ import numpy as np
import pandas as pd
from flask_babel import gettext as _
from pandas import DateOffset
from superset_core.charts.types import GenericDataType
from superset import app
from superset.common.chart_data import ChartDataResultFormat
@@ -57,7 +58,6 @@ from superset.utils.core import (
DTTM_ALIAS,
error_msg_from_exception,
FilterOperator,
GenericDataType,
get_base_axis_labels,
get_column_names_from_columns,
get_column_names_from_metrics,

View File

@@ -69,6 +69,8 @@ from sqlalchemy.sql import column, ColumnElement, literal_column, table
from sqlalchemy.sql.elements import ColumnClause, TextClause
from sqlalchemy.sql.expression import Label
from sqlalchemy.sql.selectable import Alias, TableClause
from superset_core.charts.types import GenericDataType
from superset_core.utils.backports import StrEnum
from superset import app, db, is_feature_enabled, security_manager
from superset.commands.dataset.exceptions import DatasetNotFoundError
@@ -115,7 +117,6 @@ from superset.superset_typing import (
ResultSetColumnType,
)
from superset.utils import core as utils, json
from superset.utils.backports import StrEnum
config = app.config
metadata = Model.metadata # pylint: disable=no-member
@@ -476,7 +477,7 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=
]
filtered_columns: list[Column] = []
column_types: set[utils.GenericDataType] = set()
column_types: set[GenericDataType] = set()
for column_ in data["columns"]:
generic_type = column_.get("type_generic")
if generic_type is not None:
@@ -510,7 +511,7 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=
def filter_values_handler( # pylint: disable=too-many-arguments
values: FilterValues | None,
operator: str,
target_generic_type: utils.GenericDataType,
target_generic_type: GenericDataType,
target_native_type: str | None = None,
is_list_target: bool = False,
db_engine_spec: builtins.type[BaseEngineSpec] | None = None,
@@ -524,7 +525,7 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=
return value
if (
isinstance(value, (float, int))
and target_generic_type == utils.GenericDataType.TEMPORAL
and target_generic_type == GenericDataType.TEMPORAL
and target_native_type is not None
and db_engine_spec is not None
):
@@ -537,14 +538,10 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=
if isinstance(value, str):
value = value.strip("\t\n")
if (
target_generic_type == utils.GenericDataType.NUMERIC
and operator
not in {
utils.FilterOperator.ILIKE,
utils.FilterOperator.LIKE,
}
):
if target_generic_type == GenericDataType.NUMERIC and operator not in {
utils.FilterOperator.ILIKE,
utils.FilterOperator.LIKE,
}:
# For backwards compatibility and edge cases
# where a column data type might have changed
return utils.cast_to_num(value)
@@ -552,7 +549,7 @@ class BaseDatasource(AuditMixinNullable, ImportExportMixin): # pylint: disable=
return None
if value == EMPTY_STRING:
return ""
if target_generic_type == utils.GenericDataType.BOOLEAN:
if target_generic_type == GenericDataType.BOOLEAN:
return utils.cast_to_boolean(value)
return value
@@ -891,21 +888,21 @@ class TableColumn(AuditMixinNullable, ImportExportMixin, CertificationMixin, Mod
"""
Check if the column has a boolean datatype.
"""
return self.type_generic == utils.GenericDataType.BOOLEAN
return self.type_generic == GenericDataType.BOOLEAN
@property
def is_numeric(self) -> bool:
"""
Check if the column has a numeric datatype.
"""
return self.type_generic == utils.GenericDataType.NUMERIC
return self.type_generic == GenericDataType.NUMERIC
@property
def is_string(self) -> bool:
"""
Check if the column has a string datatype.
"""
return self.type_generic == utils.GenericDataType.STRING
return self.type_generic == GenericDataType.STRING
@property
def is_temporal(self) -> bool:
@@ -917,7 +914,7 @@ class TableColumn(AuditMixinNullable, ImportExportMixin, CertificationMixin, Mod
"""
if self.is_dttm is not None:
return self.is_dttm
return self.type_generic == utils.GenericDataType.TEMPORAL
return self.type_generic == GenericDataType.TEMPORAL
@property
def database(self) -> Database:
@@ -932,9 +929,9 @@ class TableColumn(AuditMixinNullable, ImportExportMixin, CertificationMixin, Mod
return self.database.get_extra()
@property
def type_generic(self) -> utils.GenericDataType | None:
def type_generic(self) -> GenericDataType | None:
if self.is_dttm:
return utils.GenericDataType.TEMPORAL
return GenericDataType.TEMPORAL
return (
column_spec.generic_type

View File

@@ -20,7 +20,7 @@
# string to use when None values *need* to be converted to/from strings
from enum import Enum
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
USER_AGENT = "Apache Superset"

View File

@@ -57,6 +57,7 @@ from sqlalchemy.sql import literal_column, quoted_name, text
from sqlalchemy.sql.expression import ColumnClause, Select, TextAsFrom, TextClause
from sqlalchemy.types import TypeEngine
from sqlparse.tokens import CTE
from superset_core.charts.types import GenericDataType
from superset import db, sql_parse
from superset.constants import QUERY_CANCEL_KEY, TimeGrain as TimeGrainConstants
@@ -73,7 +74,7 @@ from superset.superset_typing import (
SQLAColumnType,
)
from superset.utils import core as utils, json
from superset.utils.core import ColumnSpec, GenericDataType
from superset.utils.core import ColumnSpec
from superset.utils.hashing import md5_sha_from_str
from superset.utils.json import redact_sensitive, reveal_sensitive
from superset.utils.network import is_hostname_valid, is_port_open

View File

@@ -24,10 +24,10 @@ from flask_babel import gettext as __
from sqlalchemy import Float, Integer, Numeric, String, TEXT, types
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.type_api import TypeEngine
from superset_core.charts.types import GenericDataType
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.errors import SupersetErrorType
from superset.utils.core import GenericDataType
# Regular expressions to catch custom errors
CONNECTION_ACCESS_DENIED_REGEX = re.compile(

View File

@@ -37,12 +37,12 @@ from sqlalchemy.dialects.mysql import (
TINYTEXT,
)
from sqlalchemy.engine.url import URL
from superset_core.charts.types import GenericDataType
from superset.constants import TimeGrain
from superset.db_engine_specs.base import BaseEngineSpec, BasicParametersMixin
from superset.errors import SupersetErrorType
from superset.models.sql_lab import Query
from superset.utils.core import GenericDataType
# Regular expressions to catch custom errors
CONNECTION_ACCESS_DENIED_REGEX = re.compile(

View File

@@ -39,6 +39,7 @@ from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.result import Row as ResultRow
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.expression import ColumnClause, Select
from superset_core.charts.types import GenericDataType
from superset import cache_manager, db, is_feature_enabled
from superset.common.db_query_status import QueryStatus
@@ -60,7 +61,6 @@ from superset.models.sql_types.presto_sql_types import (
from superset.result_set import destringify
from superset.superset_typing import ResultSetColumnType
from superset.utils import core as utils, json
from superset.utils.core import GenericDataType
if TYPE_CHECKING:
# prevent circular imports

View File

@@ -25,11 +25,11 @@ from flask_babel import gettext as __
from sqlalchemy import Float, Integer, Numeric, types
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.type_api import TypeEngine
from superset_core.charts.types import GenericDataType
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.errors import SupersetErrorType
from superset.models.core import Database
from superset.utils.core import GenericDataType
# Regular expressions to catch custom errors
CONNECTION_ACCESS_DENIED_REGEX = re.compile(

View File

@@ -18,8 +18,7 @@ from dataclasses import dataclass
from typing import Any, Optional
from flask_babel import lazy_gettext as _
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class SupersetErrorType(StrEnum):

View File

@@ -23,12 +23,12 @@ from typing import Any, TypedDict, Union
from uuid import UUID
from marshmallow import Schema, ValidationError
from superset_core.utils.backports import StrEnum
from superset.key_value.exceptions import (
KeyValueCodecDecodeException,
KeyValueCodecEncodeException,
)
from superset.utils.backports import StrEnum
Key = Union[int, UUID]

View File

@@ -59,6 +59,7 @@ from sqlalchemy.orm import relationship
from sqlalchemy.pool import NullPool
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.sql import ColumnElement, expression, Select
from superset_core.utils.backports import StrEnum
from superset import app, db, db_engine_specs, is_feature_enabled
from superset.commands.database.exceptions import DatabaseInvalidError
@@ -77,7 +78,6 @@ from superset.result_set import SupersetResultSet
from superset.sql_parse import Table
from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
from superset.utils import cache as cache_util, core as utils, json
from superset.utils.backports import StrEnum
from superset.utils.core import DatasourceName, get_username
from superset.utils.oauth2 import get_oauth2_access_token, OAuth2ClientConfigSchema

View File

@@ -50,6 +50,7 @@ from sqlalchemy.sql.elements import ColumnElement, literal_column, TextClause
from sqlalchemy.sql.expression import Label, Select, TextAsFrom
from sqlalchemy.sql.selectable import Alias, TableClause
from sqlalchemy_utils import UUIDType
from superset_core.charts.types import GenericDataType
from superset import app, db, is_feature_enabled
from superset.advanced_data_type.types import AdvancedDataTypeResponse
@@ -85,7 +86,6 @@ from superset.superset_typing import (
)
from superset.utils import core as utils, json
from superset.utils.core import (
GenericDataType,
get_column_name,
get_non_base_axis_columns,
get_user_id,
@@ -1135,7 +1135,7 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
def filter_values_handler( # pylint: disable=too-many-arguments
values: Optional[FilterValues],
operator: str,
target_generic_type: utils.GenericDataType,
target_generic_type: GenericDataType,
target_native_type: Optional[str] = None,
is_list_target: bool = False,
db_engine_spec: Optional[
@@ -1151,7 +1151,7 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
return value
if (
isinstance(value, (float, int))
and target_generic_type == utils.GenericDataType.TEMPORAL
and target_generic_type == GenericDataType.TEMPORAL
and target_native_type is not None
and db_engine_spec is not None
):
@@ -1164,14 +1164,10 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
if isinstance(value, str):
value = value.strip("\t\n")
if (
target_generic_type == utils.GenericDataType.NUMERIC
and operator
not in {
utils.FilterOperator.ILIKE,
utils.FilterOperator.LIKE,
}
):
if target_generic_type == GenericDataType.NUMERIC and operator not in {
utils.FilterOperator.ILIKE,
utils.FilterOperator.LIKE,
}:
# For backwards compatibility and edge cases
# where a column data type might have changed
return utils.cast_to_num(value)
@@ -1179,7 +1175,7 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
return None
if value == EMPTY_STRING:
return ""
if target_generic_type == utils.GenericDataType.BOOLEAN:
if target_generic_type == GenericDataType.BOOLEAN:
return utils.cast_to_boolean(value)
return value

View File

@@ -34,6 +34,7 @@ from sqlalchemy import (
from sqlalchemy.orm import backref, relationship
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy_utils import UUIDType
from superset_core.utils.backports import StrEnum
from superset.extensions import security_manager
from superset.models.core import Database
@@ -41,7 +42,6 @@ from superset.models.dashboard import Dashboard
from superset.models.helpers import AuditMixinNullable, ExtraJSONMixin
from superset.models.slice import Slice
from superset.reports.types import ReportScheduleExtra
from superset.utils.backports import StrEnum
from superset.utils.core import MediumText
metadata = Model.metadata # pylint: disable=no-member

View File

@@ -24,11 +24,11 @@ import numpy as np
import pandas as pd
import pyarrow as pa
from numpy.typing import NDArray
from superset_core.charts.types import GenericDataType
from superset.db_engine_specs import BaseEngineSpec
from superset.superset_typing import DbapiDescription, DbapiResult, ResultSetColumnType
from superset.utils import core as utils, json
from superset.utils.core import GenericDataType
from superset.utils import json
logger = logging.getLogger(__name__)
@@ -154,7 +154,7 @@ class SupersetResultSet:
if pa.types.is_nested(pa_data[i].type):
# TODO: revisit nested column serialization once nested types
# are added as a natively supported column type in Superset
# (superset.utils.core.GenericDataType).
# (superset_core.charts.types.GenericDataType).
stringified_arr = stringify_values(array[column])
pa_data[i] = pa.array(stringified_arr.tolist())
@@ -222,9 +222,7 @@ class SupersetResultSet:
return False
return column_spec.is_dttm
def type_generic(
self, db_type_str: Optional[str]
) -> Optional[utils.GenericDataType]:
def type_generic(self, db_type_str: Optional[str]) -> Optional[GenericDataType]:
column_spec = self.db_engine_spec.get_column_spec(db_type_str)
if column_spec is None:
return None

View File

@@ -18,8 +18,7 @@ from typing import Optional, TypedDict, Union
from flask_appbuilder.security.sqla.models import Role
from flask_login import AnonymousUserMixin
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class GuestTokenUser(TypedDict, total=False):

View File

@@ -53,6 +53,7 @@ from sqlparse.tokens import (
Wildcard,
)
from sqlparse.utils import imt
from superset_core.utils.backports import StrEnum
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import (
@@ -67,7 +68,6 @@ from superset.sql.parse import (
SQLStatement,
Table,
)
from superset.utils.backports import StrEnum
try:
from sqloxide import parse_sql as sqloxide_parse

View File

@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class LimitingFactor(StrEnum):

View File

@@ -16,15 +16,13 @@
# under the License.
from collections.abc import Sequence
from datetime import datetime
from typing import Any, Literal, Optional, TYPE_CHECKING, TypedDict, Union
from typing import Any, Literal, Optional, TypedDict, Union
from sqlalchemy.sql.type_api import TypeEngine
from superset_core.charts.types import GenericDataType
from typing_extensions import NotRequired
from werkzeug.wrappers import Response
if TYPE_CHECKING:
from superset.utils.core import GenericDataType
SQLType = Union[TypeEngine, type[TypeEngine]]
@@ -42,7 +40,7 @@ class AdhocMetricColumn(TypedDict, total=False):
is_dttm: bool
python_date_format: Optional[str]
type: str
type_generic: "GenericDataType"
type_generic: GenericDataType
verbose_name: Optional[str]
@@ -78,7 +76,7 @@ class ResultSetColumnType(TypedDict):
column_name: str
type: Optional[Union[SQLType, str]]
is_dttm: Optional[bool]
type_generic: NotRequired[Optional["GenericDataType"]]
type_generic: NotRequired[Optional[GenericDataType]]
nullable: NotRequired[Any]
default: NotRequired[Any]

View File

@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.utils.backports import StrEnum
from superset_core.utils.backports import StrEnum
class ExecutorType(StrEnum):

View File

@@ -44,7 +44,7 @@ from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate
from enum import Enum, IntEnum
from enum import Enum
from io import BytesIO
from timeit import default_timer
from types import TracebackType
@@ -71,6 +71,8 @@ from sqlalchemy.engine import Connection, Engine
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.sql.type_api import Variant
from sqlalchemy.types import TypeEngine
from superset_core.charts.types import GenericDataType
from superset_core.utils.backports import StrEnum
from typing_extensions import TypeGuard
from superset.constants import (
@@ -96,7 +98,6 @@ from superset.superset_typing import (
FormData,
Metric,
)
from superset.utils.backports import StrEnum
from superset.utils.database import get_example_database
from superset.utils.date_parser import parse_human_timedelta
from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str
@@ -131,21 +132,6 @@ class AnnotationType(StrEnum):
TIME_SERIES = "TIME_SERIES"
class GenericDataType(IntEnum):
"""
Generic database column type that fits both frontend and backend.
"""
NUMERIC = 0
STRING = 1
TEMPORAL = 2
BOOLEAN = 3
# ARRAY = 4 # Mapping all the complex data types to STRING for now
# JSON = 5 # and leaving these as a reminder.
# MAP = 6
# ROW = 7
class DatasourceType(StrEnum):
TABLE = "table"
DATASET = "dataset"

View File

@@ -22,9 +22,9 @@ from urllib.error import URLError
import numpy as np
import pandas as pd
from superset_core.charts.types import GenericDataType
from superset.utils import json
from superset.utils.core import GenericDataType
logger = logging.getLogger(__name__)

View File

@@ -18,8 +18,7 @@ import io
from typing import Any
import pandas as pd
from superset.utils.core import GenericDataType
from superset_core.charts.types import GenericDataType
def quote_formulas(df: pd.DataFrame) -> pd.DataFrame:

View File

@@ -22,10 +22,10 @@ from typing import Optional
from flask import current_app
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
from superset_core.utils.backports import StrEnum
from superset import feature_flag_manager
from superset.exceptions import SupersetException
from superset.utils.backports import StrEnum
logger = logging.getLogger(__name__)