feat!: pass datasource_type and datasource_id to form_data (#19981)

* pass datasource_type and datasource_id to form_data

* add datasource_type to delete command

* add datasource_type to delete command

* fix old keys implementation

* add more tests
This commit is contained in:
Elizabeth Thompson
2022-06-02 16:48:16 -07:00
committed by GitHub
parent a813528958
commit 32bb1ce3ff
47 changed files with 959 additions and 176 deletions

View File

@@ -56,7 +56,7 @@ def admin_id() -> int:
@pytest.fixture
def dataset_id() -> int:
def datasource() -> int:
with app.app_context() as ctx:
session: Session = ctx.app.appbuilder.get_session
dataset = (
@@ -64,24 +64,26 @@ def dataset_id() -> int:
.filter_by(table_name="wb_health_population")
.first()
)
return dataset.id
return dataset
@pytest.fixture(autouse=True)
def cache(chart_id, admin_id, dataset_id):
def cache(chart_id, admin_id, datasource):
entry: TemporaryExploreState = {
"owner": admin_id,
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": INITIAL_FORM_DATA,
}
cache_manager.explore_form_data_cache.set(KEY, entry)
def test_post(client, chart_id: int, dataset_id: int):
def test_post(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": INITIAL_FORM_DATA,
}
@@ -89,10 +91,11 @@ def test_post(client, chart_id: int, dataset_id: int):
assert resp.status_code == 201
def test_post_bad_request_non_string(client, chart_id: int, dataset_id: int):
def test_post_bad_request_non_string(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": 1234,
}
@@ -100,10 +103,11 @@ def test_post_bad_request_non_string(client, chart_id: int, dataset_id: int):
assert resp.status_code == 400
def test_post_bad_request_non_json_string(client, chart_id: int, dataset_id: int):
def test_post_bad_request_non_json_string(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": "foo",
}
@@ -111,10 +115,11 @@ def test_post_bad_request_non_json_string(client, chart_id: int, dataset_id: int
assert resp.status_code == 400
def test_post_access_denied(client, chart_id: int, dataset_id: int):
def test_post_access_denied(client, chart_id: int, datasource: SqlaTable):
login(client, "gamma")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": INITIAL_FORM_DATA,
}
@@ -122,10 +127,11 @@ def test_post_access_denied(client, chart_id: int, dataset_id: int):
assert resp.status_code == 404
def test_post_same_key_for_same_context(client, chart_id: int, dataset_id: int):
def test_post_same_key_for_same_context(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -139,11 +145,12 @@ def test_post_same_key_for_same_context(client, chart_id: int, dataset_id: int):
def test_post_different_key_for_different_context(
client, chart_id: int, dataset_id: int
client, chart_id: int, datasource: SqlaTable
):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -151,7 +158,8 @@ def test_post_different_key_for_different_context(
data = json.loads(resp.data.decode("utf-8"))
first_key = data.get("key")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"form_data": json.dumps({"test": "initial value"}),
}
resp = client.post("api/v1/explore/form_data?tab_id=1", json=payload)
@@ -160,10 +168,11 @@ def test_post_different_key_for_different_context(
assert first_key != second_key
def test_post_same_key_for_same_tab_id(client, chart_id: int, dataset_id: int):
def test_post_same_key_for_same_tab_id(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": json.dumps({"test": "initial value"}),
}
@@ -177,11 +186,12 @@ def test_post_same_key_for_same_tab_id(client, chart_id: int, dataset_id: int):
def test_post_different_key_for_different_tab_id(
client, chart_id: int, dataset_id: int
client, chart_id: int, datasource: SqlaTable
):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": json.dumps({"test": "initial value"}),
}
@@ -194,10 +204,11 @@ def test_post_different_key_for_different_tab_id(
assert first_key != second_key
def test_post_different_key_for_no_tab_id(client, chart_id: int, dataset_id: int):
def test_post_different_key_for_no_tab_id(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": INITIAL_FORM_DATA,
}
@@ -210,10 +221,11 @@ def test_post_different_key_for_no_tab_id(client, chart_id: int, dataset_id: int
assert first_key != second_key
def test_put(client, chart_id: int, dataset_id: int):
def test_put(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -221,10 +233,11 @@ def test_put(client, chart_id: int, dataset_id: int):
assert resp.status_code == 200
def test_put_same_key_for_same_tab_id(client, chart_id: int, dataset_id: int):
def test_put_same_key_for_same_tab_id(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -237,10 +250,13 @@ def test_put_same_key_for_same_tab_id(client, chart_id: int, dataset_id: int):
assert first_key == second_key
def test_put_different_key_for_different_tab_id(client, chart_id: int, dataset_id: int):
def test_put_different_key_for_different_tab_id(
client, chart_id: int, datasource: SqlaTable
):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -253,10 +269,11 @@ def test_put_different_key_for_different_tab_id(client, chart_id: int, dataset_i
assert first_key != second_key
def test_put_different_key_for_no_tab_id(client, chart_id: int, dataset_id: int):
def test_put_different_key_for_no_tab_id(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -269,10 +286,11 @@ def test_put_different_key_for_no_tab_id(client, chart_id: int, dataset_id: int)
assert first_key != second_key
def test_put_bad_request(client, chart_id: int, dataset_id: int):
def test_put_bad_request(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": 1234,
}
@@ -280,10 +298,11 @@ def test_put_bad_request(client, chart_id: int, dataset_id: int):
assert resp.status_code == 400
def test_put_bad_request_non_string(client, chart_id: int, dataset_id: int):
def test_put_bad_request_non_string(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": 1234,
}
@@ -291,10 +310,11 @@ def test_put_bad_request_non_string(client, chart_id: int, dataset_id: int):
assert resp.status_code == 400
def test_put_bad_request_non_json_string(client, chart_id: int, dataset_id: int):
def test_put_bad_request_non_json_string(client, chart_id: int, datasource: SqlaTable):
login(client, "admin")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": "foo",
}
@@ -302,10 +322,11 @@ def test_put_bad_request_non_json_string(client, chart_id: int, dataset_id: int)
assert resp.status_code == 400
def test_put_access_denied(client, chart_id: int, dataset_id: int):
def test_put_access_denied(client, chart_id: int, datasource: SqlaTable):
login(client, "gamma")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -313,10 +334,11 @@ def test_put_access_denied(client, chart_id: int, dataset_id: int):
assert resp.status_code == 404
def test_put_not_owner(client, chart_id: int, dataset_id: int):
def test_put_not_owner(client, chart_id: int, datasource: SqlaTable):
login(client, "gamma")
payload = {
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": UPDATED_FORM_DATA,
}
@@ -364,12 +386,13 @@ def test_delete_access_denied(client):
assert resp.status_code == 404
def test_delete_not_owner(client, chart_id: int, dataset_id: int, admin_id: int):
def test_delete_not_owner(client, chart_id: int, datasource: SqlaTable, admin_id: int):
another_key = "another_key"
another_owner = admin_id + 1
entry: TemporaryExploreState = {
"owner": another_owner,
"dataset_id": dataset_id,
"datasource_id": datasource.id,
"datasource_type": datasource.type,
"chart_id": chart_id,
"form_data": INITIAL_FORM_DATA,
}

View File

@@ -0,0 +1,359 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from unittest.mock import patch
import pytest
from superset import app, db, security, security_manager
from superset.commands.exceptions import DatasourceTypeInvalidError
from superset.connectors.sqla.models import SqlaTable
from superset.explore.form_data.commands.create import CreateFormDataCommand
from superset.explore.form_data.commands.delete import DeleteFormDataCommand
from superset.explore.form_data.commands.get import GetFormDataCommand
from superset.explore.form_data.commands.parameters import CommandParameters
from superset.explore.form_data.commands.update import UpdateFormDataCommand
from superset.models.slice import Slice
from superset.models.sql_lab import Query
from superset.utils.core import DatasourceType, get_example_default_schema
from superset.utils.database import get_example_database
from tests.integration_tests.base_tests import SupersetTestCase
class TestCreateFormDataCommand(SupersetTestCase):
@pytest.fixture()
def create_dataset(self):
with self.create_app().app_context():
dataset = SqlaTable(
table_name="dummy_sql_table",
database=get_example_database(),
schema=get_example_default_schema(),
sql="select 123 as intcol, 'abc' as strcol",
)
session = db.session
session.add(dataset)
session.commit()
yield dataset
# rollback
session.delete(dataset)
session.commit()
@pytest.fixture()
def create_slice(self):
with self.create_app().app_context():
session = db.session
dataset = (
session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = Slice(
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
datasource_name="tmp_perm_table",
slice_name="slice_name",
)
session.add(slice)
session.commit()
yield slice
# rollback
session.delete(slice)
session.commit()
@pytest.fixture()
def create_query(self):
with self.create_app().app_context():
session = db.session
query = Query(
sql="select 1 as foo;",
client_id="sldkfjlk",
database=get_example_database(),
)
session.add(query)
session.commit()
yield query
# rollback
session.delete(query)
session.commit()
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice")
def test_create_form_data_command(self, mock_g):
mock_g.user = security_manager.find_user("admin")
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
command = CreateFormDataCommand(args)
assert isinstance(command.run(), str)
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_create_form_data_command_invalid_type(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type="InvalidType",
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
with pytest.raises(DatasourceTypeInvalidError) as exc:
CreateFormDataCommand(create_args).run()
assert "Datasource type is invalid" in str(exc.value)
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_create_form_data_command_type_as_string(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type="table",
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
command = CreateFormDataCommand(create_args)
assert isinstance(command.run(), str)
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice")
def test_get_form_data_command(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
key = CreateFormDataCommand(create_args).run()
key_args = CommandParameters(actor=mock_g.user, key=key)
get_command = GetFormDataCommand(key_args)
cache_data = json.loads(get_command.run())
assert cache_data.get("datasource") == datasource
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_update_form_data_command(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
query = db.session.query(Query).filter_by(sql="select 1 as foo;").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
key = CreateFormDataCommand(create_args).run()
query_datasource = f"{dataset.id}__{DatasourceType.TABLE}"
update_args = CommandParameters(
actor=mock_g.user,
datasource_id=query.id,
datasource_type=DatasourceType.QUERY,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": query_datasource}),
key=key,
)
update_command = UpdateFormDataCommand(update_args)
new_key = update_command.run()
# it should return a key
assert isinstance(new_key, str)
# the updated key returned should be different from the old one
assert new_key != key
key_args = CommandParameters(actor=mock_g.user, key=key)
get_command = GetFormDataCommand(key_args)
cache_data = json.loads(get_command.run())
assert cache_data.get("datasource") == query_datasource
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_update_form_data_command_same_form_data(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
key = CreateFormDataCommand(create_args).run()
update_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
key=key,
)
update_command = UpdateFormDataCommand(update_args)
new_key = update_command.run()
# it should return a key
assert isinstance(new_key, str)
# the updated key returned should be the same as the old one
assert new_key == key
key_args = CommandParameters(actor=mock_g.user, key=key)
get_command = GetFormDataCommand(key_args)
cache_data = json.loads(get_command.run())
assert cache_data.get("datasource") == datasource
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_delete_form_data_command(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
dataset = (
db.session.query(SqlaTable).filter_by(table_name="dummy_sql_table").first()
)
slice = db.session.query(Slice).filter_by(slice_name="slice_name").first()
datasource = f"{dataset.id}__{DatasourceType.TABLE}"
create_args = CommandParameters(
actor=mock_g.user,
datasource_id=dataset.id,
datasource_type=DatasourceType.TABLE,
chart_id=slice.id,
tab_id=1,
form_data=json.dumps({"datasource": datasource}),
)
key = CreateFormDataCommand(create_args).run()
delete_args = CommandParameters(
actor=mock_g.user,
key=key,
)
delete_command = DeleteFormDataCommand(delete_args)
response = delete_command.run()
assert response == True
@patch("superset.security.manager.g")
@pytest.mark.usefixtures("create_dataset", "create_slice", "create_query")
def test_delete_form_data_command_key_expired(self, mock_g):
mock_g.user = security_manager.find_user("admin")
app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {
"REFRESH_TIMEOUT_ON_RETRIEVAL": True
}
delete_args = CommandParameters(
actor=mock_g.user,
key="some_expired_key",
)
delete_command = DeleteFormDataCommand(delete_args)
response = delete_command.run()
assert response == False