mirror of
https://github.com/apache/superset.git
synced 2026-08-04 13:02:41 +00:00
277 lines
9.7 KiB
Python
277 lines
9.7 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# pylint: disable=import-outside-toplevel, unused-argument
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from pytest_mock import MockerFixture
|
|
|
|
|
|
def test_memoized_func(mocker: MockerFixture) -> None:
|
|
"""
|
|
Test the ``memoized_func`` decorator.
|
|
"""
|
|
from superset.utils.cache import memoized_func
|
|
|
|
cache = mocker.MagicMock()
|
|
|
|
decorator = memoized_func("db:{self.id}:schema:{schema}:view_list", cache)
|
|
decorated = decorator(lambda self, schema, cache=False: 42)
|
|
|
|
self = mocker.MagicMock()
|
|
self.id = 1
|
|
|
|
# skip cache
|
|
result = decorated(self, "public", cache=False)
|
|
assert result == 42
|
|
cache.get.assert_not_called()
|
|
|
|
# check cache, no cached value
|
|
cache.get.return_value = None
|
|
result = decorated(self, "public", cache=True)
|
|
assert result == 42
|
|
cache.get.assert_called_with("db:1:schema:public:view_list")
|
|
|
|
# check cache, cached value
|
|
cache.get.return_value = 43
|
|
result = decorated(self, "public", cache=True)
|
|
assert result == 43
|
|
|
|
|
|
def _make_cache_instance(mocker: MockerFixture) -> MagicMock:
|
|
"""A cache instance whose ``.cache`` is not a ``NullCache``."""
|
|
cache_instance = mocker.MagicMock()
|
|
cache_instance.cache = object()
|
|
return cache_instance
|
|
|
|
|
|
def _patch_config(mocker: MockerFixture, **overrides: Any) -> dict[str, Any]:
|
|
config = {
|
|
"CACHE_DEFAULT_TIMEOUT": 100,
|
|
"STATS_LOGGER": mocker.MagicMock(),
|
|
"STORE_CACHE_KEYS_IN_METADATA_DB": False,
|
|
"DATA_CACHE_MAX_VALUE_SIZE": None,
|
|
}
|
|
config.update(overrides)
|
|
mocker.patch("superset.utils.cache.app.config", config)
|
|
return config
|
|
|
|
|
|
def test_set_and_log_cache_under_threshold(mocker: MockerFixture) -> None:
|
|
"""A value under DATA_CACHE_MAX_VALUE_SIZE is cached normally."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
|
|
set_and_log_cache(cache_instance, "my_key", {"df": "small"})
|
|
|
|
cache_instance.set.assert_called_once()
|
|
config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
|
|
assert (
|
|
mocker.call("skip_cache_value_too_large")
|
|
not in config["STATS_LOGGER"].incr.mock_calls
|
|
)
|
|
|
|
|
|
def test_set_and_log_cache_over_threshold(mocker: MockerFixture) -> None:
|
|
"""A value exceeding DATA_CACHE_MAX_VALUE_SIZE is not cached."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
config = _patch_config(
|
|
mocker,
|
|
DATA_CACHE_MAX_VALUE_SIZE=10,
|
|
STORE_CACHE_KEYS_IN_METADATA_DB=True,
|
|
)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
mock_session = mocker.patch("superset.utils.cache.db.session")
|
|
|
|
set_and_log_cache(
|
|
cache_instance,
|
|
"my_key",
|
|
{"df": "a value large enough to exceed the tiny threshold"},
|
|
datasource_uid="1__table",
|
|
)
|
|
|
|
cache_instance.set.assert_not_called()
|
|
config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large")
|
|
assert mocker.call("set_cache_key") not in config["STATS_LOGGER"].incr.mock_calls
|
|
mock_session.add.assert_not_called()
|
|
|
|
|
|
def test_set_and_log_cache_disabled_no_serialization(mocker: MockerFixture) -> None:
|
|
"""When the limit is None (default), no pickling overhead is incurred."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
_patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
mock_dumps = mocker.patch("superset.utils.cache.pickle.dumps")
|
|
|
|
set_and_log_cache(cache_instance, "my_key", {"df": "small"})
|
|
|
|
cache_instance.set.assert_called_once()
|
|
mock_dumps.assert_not_called()
|
|
|
|
|
|
def test_set_and_log_cache_null_cache(mocker: MockerFixture) -> None:
|
|
"""A NullCache backend short-circuits before any set."""
|
|
from flask_caching.backends import NullCache
|
|
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
_patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10)
|
|
cache_instance = mocker.MagicMock()
|
|
cache_instance.cache = NullCache()
|
|
|
|
set_and_log_cache(cache_instance, "my_key", {"df": "small"})
|
|
|
|
cache_instance.set.assert_not_called()
|
|
|
|
|
|
def test_set_and_log_cache_disabled_timeout(mocker: MockerFixture) -> None:
|
|
"""A timeout of -1 (CACHE_DISABLED_TIMEOUT) short-circuits before any set."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
_patch_config(mocker)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
|
|
set_and_log_cache(cache_instance, "my_key", {"df": "small"}, cache_timeout=-1)
|
|
|
|
cache_instance.set.assert_not_called()
|
|
|
|
|
|
def test_set_and_log_cache_equal_threshold(mocker: MockerFixture) -> None:
|
|
"""A value whose size EQUALS the threshold is still cached (guard is ``>``)."""
|
|
import pickle
|
|
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
cache_value = {"df": "boundary"}
|
|
# Compute the exact serialized size the function will see, including the
|
|
# injected ``dttm`` field, so we can set the threshold to that exact value.
|
|
dttm = "2021-01-01T00:00:00"
|
|
value = {**cache_value, "dttm": dttm}
|
|
exact_size = len(pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL))
|
|
|
|
config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=exact_size)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
# Pin the timestamp so the pickled size matches ``exact_size`` deterministically.
|
|
mock_datetime = mocker.patch("superset.utils.cache.datetime")
|
|
mock_datetime.now.return_value.replace.return_value.isoformat.return_value = dttm
|
|
|
|
set_and_log_cache(cache_instance, "my_key", cache_value)
|
|
|
|
cache_instance.set.assert_called_once()
|
|
config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
|
|
assert (
|
|
mocker.call("skip_cache_value_too_large")
|
|
not in config["STATS_LOGGER"].incr.mock_calls
|
|
)
|
|
|
|
|
|
def test_set_and_log_cache_over_threshold_no_datasource(mocker: MockerFixture) -> None:
|
|
"""Over-threshold with no datasource_uid: skipped, and no metadata-DB write."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
config = _patch_config(
|
|
mocker,
|
|
DATA_CACHE_MAX_VALUE_SIZE=10,
|
|
STORE_CACHE_KEYS_IN_METADATA_DB=True,
|
|
)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
mock_session = mocker.patch("superset.utils.cache.db.session")
|
|
|
|
set_and_log_cache(
|
|
cache_instance,
|
|
"my_key",
|
|
{"df": "a value large enough to exceed the tiny threshold"},
|
|
)
|
|
|
|
cache_instance.set.assert_not_called()
|
|
config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large")
|
|
mock_session.add.assert_not_called()
|
|
|
|
|
|
def test_set_and_log_cache_over_threshold_warns(mocker: MockerFixture) -> None:
|
|
"""The over-threshold branch emits a warning naming the key and sizes."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
_patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
mock_logger = mocker.patch("superset.utils.cache.logger")
|
|
|
|
set_and_log_cache(
|
|
cache_instance,
|
|
"my_key",
|
|
{"df": "a value large enough to exceed the tiny threshold"},
|
|
)
|
|
|
|
mock_logger.warning.assert_called_once()
|
|
warning_args = mock_logger.warning.call_args.args
|
|
assert "exceeds DATA_CACHE_MAX_VALUE_SIZE" in warning_args[0]
|
|
assert "my_key" in warning_args
|
|
|
|
|
|
def test_set_and_log_cache_under_threshold_metadata_db(mocker: MockerFixture) -> None:
|
|
"""Under-threshold with datasource_uid + metadata-DB storage writes a CacheKey."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
config = _patch_config(
|
|
mocker,
|
|
DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024,
|
|
STORE_CACHE_KEYS_IN_METADATA_DB=True,
|
|
)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
mock_session = mocker.patch("superset.utils.cache.db.session")
|
|
mock_cache_key = mocker.patch("superset.utils.cache.CacheKey")
|
|
|
|
set_and_log_cache(
|
|
cache_instance,
|
|
"my_key",
|
|
{"df": "small"},
|
|
cache_timeout=42,
|
|
datasource_uid="1__table",
|
|
)
|
|
|
|
cache_instance.set.assert_called_once()
|
|
config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
|
|
mock_cache_key.assert_called_once_with(
|
|
cache_key="my_key",
|
|
cache_timeout=42,
|
|
datasource_uid="1__table",
|
|
)
|
|
mock_session.add.assert_called_once_with(mock_cache_key.return_value)
|
|
|
|
|
|
def test_set_and_log_cache_set_failure_logs(mocker: MockerFixture) -> None:
|
|
"""A failure inside the try block is caught and logged as 'Could not cache key'."""
|
|
from superset.utils.cache import set_and_log_cache
|
|
|
|
_patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None)
|
|
cache_instance = _make_cache_instance(mocker)
|
|
boom = RuntimeError("backend down")
|
|
cache_instance.set.side_effect = boom
|
|
mock_logger = mocker.patch("superset.utils.cache.logger")
|
|
|
|
# Should not raise despite the backend failure.
|
|
set_and_log_cache(cache_instance, "my_key", {"df": "small"})
|
|
|
|
mock_logger.warning.assert_called_once_with("Could not cache key %s", "my_key")
|
|
mock_logger.exception.assert_called_once_with(boom)
|