feat(SIP-39): Async query support for charts (#11499)

* Generate JWT in Flask app

* Refactor chart data API query logic, add JWT validation and async worker

* Add redis stream implementation, refactoring

* Add chart data cache endpoint, refactor QueryContext caching

* Typing, linting, refactoring

* pytest fixes and openapi schema update

* Enforce caching be configured for async query init

* Async query processing for explore_json endpoint

* Add /api/v1/async_event endpoint

* Async frontend for dashboards [WIP]

* Chart async error message support, refactoring

* Abstract asyncEvent middleware

* Async chart loading for Explore

* Pylint fixes

* asyncEvent middleware -> TypeScript, JS linting

* Chart data API: enforce forced_cache, add tests

* Add tests for explore_json endpoints

* Add test for chart data cache enpoint (no login)

* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag

* Add tests for tasks/async_queries and address PR comments

* Bypass non-JSON result formats for async queries

* Add tests for redux middleware

* Remove debug statement

Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>

* Skip force_cached if no queryObj

* SunburstViz: don't modify self.form_data

* Fix failing annotation test

* Resolve merge/lint issues

* Reduce polling delay

* Fix new getClientErrorObject reference

* Fix flakey unit tests

* /api/v1/async_event: increment redis stream ID, add tests

* PR feedback: refactoring, configuration

* Fixup: remove debugging

* Fix typescript errors due to redux upgrade

* Update UPDATING.md

* Fix failing py tests

* asyncEvent_spec.js -> asyncEvent_spec.ts

* Refactor flakey Python 3.7 mock assertions

* Fix another shared state issue in Py tests

* Use 'sub' claim in JWT for user_id

* Refactor async middleware config

* Fixup: restore FeatureFlag boolean type

Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>

superset/utils/cache.py

@@ -14,16 +14,65 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import hashlib
+import json
 import logging
 from datetime import datetime, timedelta
 from functools import wraps
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 from flask import current_app as app, request
 from flask_caching import Cache
 from werkzeug.wrappers.etag import ETagResponseMixin
 
+from superset import db
 from superset.extensions import cache_manager
+from superset.models.cache import CacheKey
+from superset.stats_logger import BaseStatsLogger
+from superset.utils.core import json_int_dttm_ser
+
+config = app.config  # type: ignore
+stats_logger: BaseStatsLogger = config["STATS_LOGGER"]
+logger = logging.getLogger(__name__)
+
+# TODO: DRY up cache key code
+def json_dumps(obj: Any, sort_keys: bool = False) -> str:
+    return json.dumps(obj, default=json_int_dttm_ser, sort_keys=sort_keys)
+
+
+def generate_cache_key(values_dict: Dict[str, Any], key_prefix: str = "") -> str:
+    json_data = json_dumps(values_dict, sort_keys=True)
+    hash_str = hashlib.md5(json_data.encode("utf-8")).hexdigest()
+    return f"{key_prefix}{hash_str}"
+
+
+def set_and_log_cache(
+    cache_instance: Cache,
+    cache_key: str,
+    cache_value: Dict[str, Any],
+    cache_timeout: Optional[int] = None,
+    datasource_uid: Optional[str] = None,
+) -> None:
+    timeout = cache_timeout if cache_timeout else config["CACHE_DEFAULT_TIMEOUT"]
+    try:
+        dttm = datetime.utcnow().isoformat().split(".")[0]
+        value = {**cache_value, "dttm": dttm}
+        cache_instance.set(cache_key, value, timeout=timeout)
+        stats_logger.incr("set_cache_key")
+
+        if datasource_uid and config["STORE_CACHE_KEYS_IN_METADATA_DB"]:
+            ck = CacheKey(
+                cache_key=cache_key,
+                cache_timeout=cache_timeout,
+                datasource_uid=datasource_uid,
+            )
+            db.session.add(ck)
+    except Exception as ex:  # pylint: disable=broad-except
+        # cache.set call can fail if the backend is down or if
+        # the key is too large or whatever other reasons
+        logger.warning("Could not cache key %s", cache_key)
+        logger.exception(ex)
+
 
 # If a user sets `max_age` to 0, for long the browser should cache the
 # resource? Flask-Caching will cache forever, but for the HTTP header we need