chore(api v1): Deprecate datasource/save and datasource/get endpoints (#23678)

This commit is contained in:
Jack Fragassi
2023-04-18 17:51:24 -07:00
committed by GitHub
parent 818a1d482b
commit 44557f5a23
23 changed files with 242 additions and 70 deletions

View File

@@ -20,7 +20,7 @@ import { DatasourceType } from '@superset-ui/core';
import { Dataset } from './types';
export const TestDataset: Dataset = {
column_format: {},
column_formats: {},
columns: [
{
advanced_data_type: undefined,

View File

@@ -67,7 +67,7 @@ export interface Dataset {
type: DatasourceType;
columns: ColumnMeta[];
metrics: Metric[];
column_format: Record<string, string>;
column_formats: Record<string, string>;
verbose_map: Record<string, string>;
main_dttm_col: string;
// eg. ['["ds", true]', 'ds [asc]']

View File

@@ -42,7 +42,7 @@ describe('columnChoices()', () => {
},
],
verbose_map: {},
column_format: { fiz: 'NUMERIC', about: 'STRING', foo: 'DATE' },
column_formats: { fiz: 'NUMERIC', about: 'STRING', foo: 'DATE' },
datasource_name: 'my_datasource',
description: 'this is my datasource',
}),

View File

@@ -39,7 +39,7 @@ describe('defineSavedMetrics', () => {
time_grain_sqla: 'P1D',
columns: [],
verbose_map: {},
column_format: {},
column_formats: {},
datasource_name: 'my_datasource',
description: 'this is my datasource',
};

View File

@@ -49,7 +49,7 @@ const datasourceData = {
const DATASOURCES_ENDPOINT =
'glob:*/api/v1/dataset/?q=(order_column:changed_on_delta_humanized,order_direction:desc,page:0,page_size:25)';
const DATASOURCE_ENDPOINT = `glob:*/datasource/get/${datasourceData.type}/${datasourceData.id}`;
const DATASOURCE_ENDPOINT = `glob:*/api/v1/dataset/${datasourceData.id}`;
const DATASOURCE_PAYLOAD = { new: 'data' };
const INFO_ENDPOINT = 'glob:*/api/v1/dataset/_info?*';
@@ -112,6 +112,6 @@ describe('ChangeDatasourceModal', () => {
});
await waitForComponentToPaint(wrapper);
expect(fetchMock.calls(/datasource\/get\/table\/7/)).toHaveLength(1);
expect(fetchMock.calls(/api\/v1\/dataset\/7/)).toHaveLength(1);
});
});

View File

@@ -173,10 +173,12 @@ const ChangeDatasourceModal: FunctionComponent<ChangeDatasourceModalProps> = ({
const handleChangeConfirm = () => {
SupersetClient.get({
endpoint: `/datasource/get/${confirmedDataset?.type}/${confirmedDataset?.id}/`,
endpoint: `/api/v1/dataset/${confirmedDataset?.id}`,
})
.then(({ json }) => {
onDatasourceSave(json);
// eslint-disable-next-line no-param-reassign
json.result.type = 'table';
onDatasourceSave(json.result);
onChange(`${confirmedDataset?.id}__table`);
})
.catch(response => {

View File

@@ -40,7 +40,8 @@ const datasource = mockDatasource['7__table'];
const SAVE_ENDPOINT = 'glob:*/api/v1/dataset/7';
const SAVE_PAYLOAD = { new: 'data' };
const SAVE_DATASOURCE_ENDPOINT = 'glob:*/datasource/save/';
const SAVE_DATASOURCE_ENDPOINT = 'glob:*/api/v1/dataset/7';
const GET_DATASOURCE_ENDPOINT = SAVE_DATASOURCE_ENDPOINT;
const mockedProps = {
datasource,
@@ -96,7 +97,8 @@ describe('DatasourceModal', () => {
it('saves on confirm', async () => {
const callsP = fetchMock.post(SAVE_ENDPOINT, SAVE_PAYLOAD);
fetchMock.post(SAVE_DATASOURCE_ENDPOINT, {});
fetchMock.put(SAVE_DATASOURCE_ENDPOINT, {});
fetchMock.get(GET_DATASOURCE_ENDPOINT, {});
act(() => {
wrapper
.find('button[data-test="datasource-modal-save"]')
@@ -111,7 +113,11 @@ describe('DatasourceModal', () => {
okButton.simulate('click');
});
await waitForComponentToPaint(wrapper);
const expected = ['http://localhost/datasource/save/'];
// one call to PUT, then one to GET
const expected = [
'http://localhost/api/v1/dataset/7',
'http://localhost/api/v1/dataset/7',
];
expect(callsP._calls.map(call => call[0])).toEqual(
expected,
); /* eslint no-underscore-dangle: 0 */

View File

@@ -96,39 +96,81 @@ const DatasourceModal: FunctionComponent<DatasourceModalProps> = ({
currentDatasource.schema;
setIsSaving(true);
SupersetClient.post({
endpoint: '/datasource/save/',
postPayload: {
data: {
...currentDatasource,
cache_timeout:
currentDatasource.cache_timeout === ''
? null
: currentDatasource.cache_timeout,
schema,
metrics: currentDatasource?.metrics?.map(
(metric: Record<string, unknown>) => ({
...metric,
SupersetClient.put({
endpoint: `/api/v1/dataset/${currentDatasource.id}`,
jsonPayload: {
table_name: currentDatasource.table_name,
database_id: currentDatasource.database?.id,
sql: currentDatasource.sql,
filter_select_enabled: currentDatasource.filter_select_enabled,
fetch_values_predicate: currentDatasource.fetch_values_predicate,
schema,
description: currentDatasource.description,
main_dttm_col: currentDatasource.main_dttm_col,
offset: currentDatasource.offset,
default_endpoint: currentDatasource.default_endpoint,
cache_timeout:
currentDatasource.cache_timeout === ''
? null
: currentDatasource.cache_timeout,
is_sqllab_view: currentDatasource.is_sqllab_view,
template_params: currentDatasource.template_params,
extra: currentDatasource.extra,
is_managed_externally: currentDatasource.is_managed_externally,
external_url: currentDatasource.external_url,
metrics: currentDatasource?.metrics?.map(
(metric: Record<string, unknown>) => {
const metricBody: any = {
expression: metric.expression,
description: metric.description,
metric_name: metric.metric_name,
metric_type: metric.metric_type,
d3format: metric.d3format,
verbose_name: metric.verbose_name,
warning_text: metric.warning_text,
uuid: metric.uuid,
extra: buildExtraJsonObject(metric),
}),
),
columns: currentDatasource?.columns?.map(
(column: Record<string, unknown>) => ({
...column,
extra: buildExtraJsonObject(column),
}),
),
type: currentDatasource.type || currentDatasource.datasource_type,
owners: currentDatasource.owners.map(
(o: Record<string, number>) => o.value || o.id,
),
},
};
if (!Number.isNaN(Number(metric.id))) {
metricBody.id = metric.id;
}
return metricBody;
},
),
columns: currentDatasource?.columns?.map(
(column: Record<string, unknown>) => ({
id: column.id,
column_name: column.column_name,
type: column.type,
advanced_data_type: column.advanced_data_type,
verbose_name: column.verbose_name,
description: column.description,
expression: column.expression,
filterable: column.filterable,
groupby: column.groupby,
is_active: column.is_active,
is_dttm: column.is_dttm,
python_date_format: column.python_date_format,
uuid: column.uuid,
extra: buildExtraJsonObject(column),
}),
),
owners: currentDatasource.owners.map(
(o: Record<string, number>) => o.value || o.id,
),
},
})
.then(({ json }) => {
.then(() => {
addSuccessToast(t('The dataset has been saved'));
return SupersetClient.get({
endpoint: `/api/v1/dataset/${currentDatasource?.id}`,
});
})
.then(({ json }) => {
// eslint-disable-next-line no-param-reassign
json.result.type = 'table';
onDatasourceSave({
...json,
...json.result,
owners: currentDatasource.owners,
});
onHide();

View File

@@ -28,7 +28,7 @@ export const PLACEHOLDER_DATASOURCE: Datasource = {
columns: [],
column_types: [],
metrics: [],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '',
description: '',

View File

@@ -34,7 +34,7 @@ const CURRENT_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']
@@ -47,7 +47,7 @@ const NEW_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']

View File

@@ -69,9 +69,20 @@ const createProps = (overrides: JsonObject = {}) => ({
});
async function openAndSaveChanges(datasource: any) {
fetchMock.post('glob:*/datasource/save/', datasource, {
overwriteRoutes: true,
});
fetchMock.put(
'glob:*/api/v1/dataset/*',
{},
{
overwriteRoutes: true,
},
);
fetchMock.get(
'glob:*/api/v1/dataset/*',
{ result: datasource },
{
overwriteRoutes: true,
},
);
userEvent.click(screen.getByTestId('datasource-menu-trigger'));
userEvent.click(await screen.findByTestId('edit-dataset'));
userEvent.click(await screen.findByTestId('datasource-modal-save'));
@@ -154,7 +165,7 @@ test('Should show SQL Lab for sql_lab role', async () => {
test('Click on Swap dataset option', async () => {
const props = createProps();
SupersetClientGet.mockImplementation(
SupersetClientGet.mockImplementationOnce(
async ({ endpoint }: { endpoint: string }) => {
if (endpoint.includes('_info')) {
return {
@@ -182,7 +193,7 @@ test('Click on Swap dataset option', async () => {
test('Click on Edit dataset', async () => {
const props = createProps();
SupersetClientGet.mockImplementation(
SupersetClientGet.mockImplementationOnce(
async () => ({ json: { result: [] } } as any),
);
render(<DatasourceControl {...props} />, {
@@ -206,7 +217,7 @@ test('Edit dataset should be disabled when user is not admin', async () => {
// @ts-expect-error
props.user.roles = {};
props.datasource.owners = [];
SupersetClientGet.mockImplementation(
SupersetClientGet.mockImplementationOnce(
async () => ({ json: { result: [] } } as any),
);

View File

@@ -51,7 +51,7 @@ describe('controlUtils', () => {
type: DatasourceType.Table,
columns: [{ column_name: 'a' }],
metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: '1__table',

View File

@@ -34,7 +34,7 @@ const sampleDatasource: Dataset = {
{ column_name: 'sample_column_4' },
],
metrics: [{ metric_name: 'saved_metric_2' }],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: 'Sample Dataset',

View File

@@ -135,7 +135,7 @@ export const exploreInitialData: ExplorePageInitialData = {
type: DatasourceType.Table,
columns: [{ column_name: 'a' }],
metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: '8__table',
@@ -153,7 +153,7 @@ export const fallbackExploreInitialData: ExplorePageInitialData = {
type: DatasourceType.Table,
columns: [],
metrics: [],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '',
owners: [],

View File

@@ -25,7 +25,7 @@ const TEST_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
column_format: {},
column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']

View File

@@ -19,7 +19,18 @@ from __future__ import annotations
import json
from datetime import datetime
from enum import Enum
from typing import Any, Dict, Hashable, List, Optional, Set, Type, TYPE_CHECKING, Union
from typing import (
Any,
Dict,
Hashable,
List,
Optional,
Set,
Tuple,
Type,
TYPE_CHECKING,
Union,
)
from flask_appbuilder.security.sqla.models import User
from flask_babel import gettext as __
@@ -242,26 +253,33 @@ class BaseDatasource(
pass
@property
def data(self) -> Dict[str, Any]:
"""Data representation of the datasource sent to the frontend"""
order_by_choices = []
def order_by_choices(self) -> List[Tuple[str, str]]:
choices = []
# self.column_names return sorted column_names
for column_name in self.column_names:
column_name = str(column_name or "")
order_by_choices.append(
choices.append(
(json.dumps([column_name, True]), f"{column_name} " + __("[asc]"))
)
order_by_choices.append(
choices.append(
(json.dumps([column_name, False]), f"{column_name} " + __("[desc]"))
)
return choices
verbose_map = {"__timestamp": "Time"}
verbose_map.update(
@property
def verbose_map(self) -> Dict[str, str]:
verb_map = {"__timestamp": "Time"}
verb_map.update(
{o.metric_name: o.verbose_name or o.metric_name for o in self.metrics}
)
verbose_map.update(
verb_map.update(
{o.column_name: o.verbose_name or o.column_name for o in self.columns}
)
return verb_map
@property
def data(self) -> Dict[str, Any]:
"""Data representation of the datasource sent to the frontend"""
return {
# simple fields
"id": self.id,
@@ -288,9 +306,9 @@ class BaseDatasource(
"columns": [o.data for o in self.columns],
"metrics": [o.data for o in self.metrics],
# TODO deprecate, move logic to JS
"order_by_choices": order_by_choices,
"order_by_choices": self.order_by_choices,
"owners": [owner.id for owner in self.owners],
"verbose_map": verbose_map,
"verbose_map": self.verbose_map,
"select_star": self.select_star,
}

View File

@@ -785,14 +785,20 @@ class SqlaTable(
check = config["DATASET_HEALTH_CHECK"]
return check(self) if check else None
@property
def granularity_sqla(self) -> List[Tuple[Any, Any]]:
return utils.choicify(self.dttm_cols)
@property
def time_grain_sqla(self) -> List[Tuple[Any, Any]]:
return [(g.duration, g.name) for g in self.database.grains() or []]
@property
def data(self) -> Dict[str, Any]:
data_ = super().data
if self.type == "table":
data_["granularity_sqla"] = utils.choicify(self.dttm_cols)
data_["time_grain_sqla"] = [
(g.duration, g.name) for g in self.database.grains() or []
]
data_["granularity_sqla"] = self.granularity_sqla
data_["time_grain_sqla"] = self.time_grain_sqla
data_["main_dttm_col"] = self.main_dttm_col
data_["fetch_values_predicate"] = self.fetch_values_predicate
data_["template_params"] = self.template_params

View File

@@ -195,6 +195,14 @@ class DatasetRestApi(BaseSupersetModelRestApi):
"database.backend",
"columns.advanced_data_type",
"is_managed_externally",
"uid",
"datasource_name",
"name",
"column_formats",
"granularity_sqla",
"time_grain_sqla",
"order_by_choices",
"verbose_map",
]
add_model_schema = DatasetPostSchema()
edit_model_schema = DatasetPutSchema()

View File

@@ -61,6 +61,23 @@ class DatasetExistsValidationError(ValidationError):
)
class DatasetEndpointUnsafeValidationError(ValidationError):
"""
Marshmallow validation error for unsafe dataset default endpoint
"""
def __init__(self) -> None:
super().__init__(
[
_(
"The submitted URL is not considered safe,"
" only use URLs with the same domain as Superset."
)
],
field_name="default_endpoint",
)
class DatasetColumnNotFoundValidationError(ValidationError):
"""
Marshmallow validation error when dataset column for update does not exist

View File

@@ -18,6 +18,7 @@ import logging
from collections import Counter
from typing import Any, Dict, List, Optional
from flask import current_app
from flask_appbuilder.models.sqla import Model
from marshmallow import ValidationError
@@ -30,6 +31,7 @@ from superset.datasets.commands.exceptions import (
DatasetColumnNotFoundValidationError,
DatasetColumnsDuplicateValidationError,
DatasetColumnsExistsValidationError,
DatasetEndpointUnsafeValidationError,
DatasetExistsValidationError,
DatasetForbiddenError,
DatasetInvalidError,
@@ -41,6 +43,7 @@ from superset.datasets.commands.exceptions import (
)
from superset.datasets.dao import DatasetDAO
from superset.exceptions import SupersetSecurityException
from superset.utils.urls import is_safe_url
logger = logging.getLogger(__name__)
@@ -101,6 +104,14 @@ class UpdateDatasetCommand(UpdateMixin, BaseCommand):
self._properties["owners"] = owners
except ValidationError as ex:
exceptions.append(ex)
# Validate default URL safety
default_endpoint = self._properties.get("default_endpoint")
if (
default_endpoint
and not is_safe_url(default_endpoint)
and current_app.config["PREVENT_UNSAFE_DEFAULT_URLS_ON_DATASET"]
):
exceptions.append(DatasetEndpointUnsafeValidationError())
# Validate columns
columns = self._properties.get("columns")

View File

@@ -55,7 +55,7 @@ class DatasetColumnsPutSchema(Schema):
extra = fields.String(allow_none=True)
filterable = fields.Boolean()
groupby = fields.Boolean()
is_active = fields.Boolean()
is_active = fields.Boolean(allow_none=True)
is_dttm = fields.Boolean(default=False)
python_date_format = fields.String(
allow_none=True, validate=[Length(1, 255), validate_python_date_format]

View File

@@ -44,6 +44,7 @@ from superset.utils.urls import is_safe_url
from superset.views.base import (
api,
BaseSupersetView,
deprecated,
handle_api_exception,
json_error_response,
)
@@ -69,6 +70,7 @@ class Datasource(BaseSupersetView):
@has_access_api
@api
@handle_api_exception
@deprecated(new_target="/api/v1/dataset/<int:pk>")
def save(self) -> FlaskResponse:
data = request.form.get("data")
if not isinstance(data, str):
@@ -133,6 +135,7 @@ class Datasource(BaseSupersetView):
@has_access_api
@api
@handle_api_exception
@deprecated(new_target="/api/v1/dataset/<int:pk>")
def get(self, datasource_type: str, datasource_id: int) -> FlaskResponse:
datasource = DatasourceDAO.get_datasource(
db.session, DatasourceType(datasource_type), datasource_id

View File

@@ -19,7 +19,7 @@ import json
import unittest
from io import BytesIO
from typing import List, Optional
from unittest.mock import patch
from unittest.mock import ANY, patch
from zipfile import is_zipfile, ZipFile
import prison
@@ -347,6 +347,28 @@ class TestDatasetApi(SupersetTestCase):
"sql": None,
"table_name": "energy_usage",
"template_params": None,
"uid": "2__table",
"datasource_name": "energy_usage",
"name": f"{get_example_default_schema()}.energy_usage",
"column_formats": {},
"granularity_sqla": [],
"time_grain_sqla": ANY,
"order_by_choices": [
['["source", true]', "source [asc]"],
['["source", false]', "source [desc]"],
['["target", true]', "target [asc]"],
['["target", false]', "target [desc]"],
['["value", true]', "value [asc]"],
['["value", false]', "value [desc]"],
],
"verbose_map": {
"__timestamp": "Time",
"count": "COUNT(*)",
"source": "source",
"sum__value": "sum__value",
"target": "target",
"value": "value",
},
}
if response["result"]["database"]["backend"] not in ("presto", "hive"):
assert {
@@ -1350,6 +1372,32 @@ class TestDatasetApi(SupersetTestCase):
db.session.delete(ab_user)
db.session.commit()
def test_update_dataset_unsafe_default_endpoint(self):
"""
Dataset API: Test unsafe default endpoint
"""
if backend() == "sqlite":
return
dataset = self.insert_default_dataset()
self.login(username="admin")
uri = f"api/v1/dataset/{dataset.id}"
table_data = {"default_endpoint": "http://www.google.com"}
rv = self.client.put(uri, json=table_data)
data = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 422
expected_response = {
"message": {
"default_endpoint": [
"The submitted URL is not considered safe,"
" only use URLs with the same domain as Superset."
]
}
}
assert data == expected_response
db.session.delete(dataset)
db.session.commit()
@patch("superset.datasets.dao.DatasetDAO.update")
def test_update_dataset_sqlalchemy_error(self, mock_dao_update):
"""