[SQL Lab] Async query results serialization with MessagePack and PyArrow (#8069)

* Add support for msgpack results_backend serialization

* Serialize DataFrame with PyArrow rather than JSON

* Adjust dependencies, de-lint

* Add tests for (de)serialization methods

* Add MessagePack config info to Installation docs

* Enable msgpack/arrow serialization by default

* [Fix] Prevent msgpack serialization on synchronous queries

* Add type annotations
This commit is contained in:
Rob DiCiuccio
2019-08-27 14:23:40 -07:00
committed by Maxime Beauchemin
parent 56566c2645
commit 7595d9e5fd
13 changed files with 362 additions and 28 deletions

View File

@@ -33,7 +33,7 @@ import smtplib
import sys
from time import struct_time
import traceback
from typing import List, NamedTuple, Optional, Tuple
from typing import List, NamedTuple, Optional, Tuple, Union
from urllib.parse import unquote_plus
import uuid
import zlib
@@ -803,12 +803,12 @@ def zlib_compress(data):
return zlib.compress(data)
def zlib_decompress_to_string(blob):
def zlib_decompress(blob: bytes, decode: Optional[bool] = True) -> Union[bytes, str]:
"""
Decompress things to a string in a py2/3 safe fashion
>>> json_str = '{"test": 1}'
>>> blob = zlib_compress(json_str)
>>> got_str = zlib_decompress_to_string(blob)
>>> got_str = zlib_decompress(blob)
>>> got_str == json_str
True
"""
@@ -817,7 +817,7 @@ def zlib_decompress_to_string(blob):
decompressed = zlib.decompress(blob)
else:
decompressed = zlib.decompress(bytes(blob, "utf-8"))
return decompressed.decode("utf-8")
return decompressed.decode("utf-8") if decode else decompressed
return zlib.decompress(blob)