# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # pylint: disable=import-outside-toplevel, unused-argument from typing import Any from unittest.mock import MagicMock from pytest_mock import MockerFixture def test_memoized_func(mocker: MockerFixture) -> None: """ Test the ``memoized_func`` decorator. """ from superset.utils.cache import memoized_func cache = mocker.MagicMock() decorator = memoized_func("db:{self.id}:schema:{schema}:view_list", cache) decorated = decorator(lambda self, schema, cache=False: 42) self = mocker.MagicMock() self.id = 1 # skip cache result = decorated(self, "public", cache=False) assert result == 42 cache.get.assert_not_called() # check cache, no cached value cache.get.return_value = None result = decorated(self, "public", cache=True) assert result == 42 cache.get.assert_called_with("db:1:schema:public:view_list") # check cache, cached value cache.get.return_value = 43 result = decorated(self, "public", cache=True) assert result == 43 def _make_cache_instance(mocker: MockerFixture) -> MagicMock: """A cache instance whose ``.cache`` is not a ``NullCache``.""" cache_instance = mocker.MagicMock() cache_instance.cache = object() return cache_instance def _patch_config(mocker: MockerFixture, **overrides: Any) -> dict[str, Any]: config = { "CACHE_DEFAULT_TIMEOUT": 100, "STATS_LOGGER": mocker.MagicMock(), "STORE_CACHE_KEYS_IN_METADATA_DB": False, "DATA_CACHE_MAX_VALUE_SIZE": None, } config.update(overrides) mocker.patch("superset.utils.cache.app.config", config) return config def test_set_and_log_cache_under_threshold(mocker: MockerFixture) -> None: """A value under DATA_CACHE_MAX_VALUE_SIZE is cached normally.""" from superset.utils.cache import set_and_log_cache config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024) cache_instance = _make_cache_instance(mocker) set_and_log_cache(cache_instance, "my_key", {"df": "small"}) cache_instance.set.assert_called_once() config["STATS_LOGGER"].incr.assert_any_call("set_cache_key") assert ( mocker.call("skip_cache_value_too_large") not in config["STATS_LOGGER"].incr.mock_calls ) def test_set_and_log_cache_over_threshold(mocker: MockerFixture) -> None: """A value exceeding DATA_CACHE_MAX_VALUE_SIZE is not cached.""" from superset.utils.cache import set_and_log_cache config = _patch_config( mocker, DATA_CACHE_MAX_VALUE_SIZE=10, STORE_CACHE_KEYS_IN_METADATA_DB=True, ) cache_instance = _make_cache_instance(mocker) mock_session = mocker.patch("superset.utils.cache.db.session") set_and_log_cache( cache_instance, "my_key", {"df": "a value large enough to exceed the tiny threshold"}, datasource_uid="1__table", ) cache_instance.set.assert_not_called() config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large") assert mocker.call("set_cache_key") not in config["STATS_LOGGER"].incr.mock_calls mock_session.add.assert_not_called() def test_set_and_log_cache_disabled_no_serialization(mocker: MockerFixture) -> None: """When the limit is None (default), no pickling overhead is incurred.""" from superset.utils.cache import set_and_log_cache _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None) cache_instance = _make_cache_instance(mocker) mock_dumps = mocker.patch("superset.utils.cache.pickle.dumps") set_and_log_cache(cache_instance, "my_key", {"df": "small"}) cache_instance.set.assert_called_once() mock_dumps.assert_not_called() def test_set_and_log_cache_null_cache(mocker: MockerFixture) -> None: """A NullCache backend short-circuits before any set.""" from flask_caching.backends import NullCache from superset.utils.cache import set_and_log_cache _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10) cache_instance = mocker.MagicMock() cache_instance.cache = NullCache() set_and_log_cache(cache_instance, "my_key", {"df": "small"}) cache_instance.set.assert_not_called() def test_set_and_log_cache_disabled_timeout(mocker: MockerFixture) -> None: """A timeout of -1 (CACHE_DISABLED_TIMEOUT) short-circuits before any set.""" from superset.utils.cache import set_and_log_cache _patch_config(mocker) cache_instance = _make_cache_instance(mocker) set_and_log_cache(cache_instance, "my_key", {"df": "small"}, cache_timeout=-1) cache_instance.set.assert_not_called() def test_set_and_log_cache_equal_threshold(mocker: MockerFixture) -> None: """A value whose size EQUALS the threshold is still cached (guard is ``>``).""" import pickle from superset.utils.cache import set_and_log_cache cache_value = {"df": "boundary"} # Compute the exact serialized size the function will see, including the # injected ``dttm`` field, so we can set the threshold to that exact value. dttm = "2021-01-01T00:00:00" value = {**cache_value, "dttm": dttm} exact_size = len(pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)) config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=exact_size) cache_instance = _make_cache_instance(mocker) # Pin the timestamp so the pickled size matches ``exact_size`` deterministically. mock_datetime = mocker.patch("superset.utils.cache.datetime") mock_datetime.now.return_value.replace.return_value.isoformat.return_value = dttm set_and_log_cache(cache_instance, "my_key", cache_value) cache_instance.set.assert_called_once() config["STATS_LOGGER"].incr.assert_any_call("set_cache_key") assert ( mocker.call("skip_cache_value_too_large") not in config["STATS_LOGGER"].incr.mock_calls ) def test_set_and_log_cache_over_threshold_no_datasource(mocker: MockerFixture) -> None: """Over-threshold with no datasource_uid: skipped, and no metadata-DB write.""" from superset.utils.cache import set_and_log_cache config = _patch_config( mocker, DATA_CACHE_MAX_VALUE_SIZE=10, STORE_CACHE_KEYS_IN_METADATA_DB=True, ) cache_instance = _make_cache_instance(mocker) mock_session = mocker.patch("superset.utils.cache.db.session") set_and_log_cache( cache_instance, "my_key", {"df": "a value large enough to exceed the tiny threshold"}, ) cache_instance.set.assert_not_called() config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large") mock_session.add.assert_not_called() def test_set_and_log_cache_over_threshold_warns(mocker: MockerFixture) -> None: """The over-threshold branch emits a warning naming the key and sizes.""" from superset.utils.cache import set_and_log_cache _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10) cache_instance = _make_cache_instance(mocker) mock_logger = mocker.patch("superset.utils.cache.logger") set_and_log_cache( cache_instance, "my_key", {"df": "a value large enough to exceed the tiny threshold"}, ) mock_logger.warning.assert_called_once() warning_args = mock_logger.warning.call_args.args assert "exceeds DATA_CACHE_MAX_VALUE_SIZE" in warning_args[0] assert "my_key" in warning_args def test_set_and_log_cache_under_threshold_metadata_db(mocker: MockerFixture) -> None: """Under-threshold with datasource_uid + metadata-DB storage writes a CacheKey.""" from superset.utils.cache import set_and_log_cache config = _patch_config( mocker, DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024, STORE_CACHE_KEYS_IN_METADATA_DB=True, ) cache_instance = _make_cache_instance(mocker) mock_session = mocker.patch("superset.utils.cache.db.session") mock_cache_key = mocker.patch("superset.utils.cache.CacheKey") set_and_log_cache( cache_instance, "my_key", {"df": "small"}, cache_timeout=42, datasource_uid="1__table", ) cache_instance.set.assert_called_once() config["STATS_LOGGER"].incr.assert_any_call("set_cache_key") mock_cache_key.assert_called_once_with( cache_key="my_key", cache_timeout=42, datasource_uid="1__table", ) mock_session.add.assert_called_once_with(mock_cache_key.return_value) def test_set_and_log_cache_set_failure_logs(mocker: MockerFixture) -> None: """A failure inside the try block is caught and logged as 'Could not cache key'.""" from superset.utils.cache import set_and_log_cache _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None) cache_instance = _make_cache_instance(mocker) boom = RuntimeError("backend down") cache_instance.set.side_effect = boom mock_logger = mocker.patch("superset.utils.cache.logger") # Should not raise despite the backend failure. set_and_log_cache(cache_instance, "my_key", {"df": "small"}) mock_logger.warning.assert_called_once_with("Could not cache key %s", "my_key") mock_logger.exception.assert_called_once_with(boom)