QT/superset2
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2026-04-19 16:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(streaming): Streaming CSV uploads for over 100k records for constant memory usage (#35478)

Browse Source
This commit is contained in:
amaannawab923
2025-11-20 22:46:59 +05:30
committed by GitHub
parent 6d359161bb
commit 35f156a1e1
27 changed files with 3096 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
tests/integration_tests/charts/data/api_tests.py
View File

@@ -803,13 +803,13 @@ class TestPostChartDataApi(BaseTestChartDataApi):
payload_with_force["force"] = True
self.post_assert_metric(CHART_DATA_URI, payload_with_force, "data")
# Check that is_cached was logged as None (not from cache)
# Check that is_cached was logged as [None] (not from cache)
call_kwargs = mock_event_logger.call_args[1]
records = call_kwargs.get("records", [])
assert len(records) > 0
# is_cached should be None when force=True (bypasses cache)
# is_cached should be [None] when force=True (bypasses cache)
assert "is_cached" in records[0]
assert records[0]["is_cached"] is None
assert records[0]["is_cached"] == [None]
# Reset mock for second request
mock_event_logger.reset_mock()
@@ -819,12 +819,12 @@ class TestPostChartDataApi(BaseTestChartDataApi):
payload_without_force["force"] = False
self.post_assert_metric(CHART_DATA_URI, payload_without_force, "data")
# Check that is_cached was logged as True (from cache)
# Check that is_cached was logged as [True] (from cache)
call_kwargs = mock_event_logger.call_args[1]
records = call_kwargs.get("records", [])
assert len(records) > 0
# is_cached should be True when retrieved from cache
assert records[0]["is_cached"] is True
# is_cached should be [True] when retrieved from cache
assert records[0]["is_cached"] == [True]
@with_feature_flags(GLOBAL_ASYNC_QUERIES=True)
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
@@ -1323,14 +1323,14 @@ class TestGetChartDataApi(BaseTestChartDataApi):
# First request - should not be cached (force=true bypasses cache)
self.get_assert_metric(f"api/v1/chart/{chart.id}/data/?force=true", "get_data")
# Check that is_cached was logged as None (not from cache)
# Check that is_cached was logged as [None] (not from cache)
call_kwargs = mock_event_logger.call_args[1]
records = call_kwargs.get("records", [])
assert len(records) > 0
# is_cached should be None when force=true (bypasses cache)
# The field should exist but be None
# is_cached should be [None] when force=true (bypasses cache)
# The field should exist but contain [None]
assert "is_cached" in records[0]
assert records[0]["is_cached"] is None
assert records[0]["is_cached"] == [None]
# Reset mock for second request
mock_event_logger.reset_mock()
@@ -1338,12 +1338,12 @@ class TestGetChartDataApi(BaseTestChartDataApi):
# Second request - should be cached
self.get_assert_metric(f"api/v1/chart/{chart.id}/data/", "get_data")
# Check that is_cached was logged as True (from cache)
# Check that is_cached was logged as [True] (from cache)
call_kwargs = mock_event_logger.call_args[1]
records = call_kwargs.get("records", [])
assert len(records) > 0
# is_cached should be True when retrieved from cache
assert records[0]["is_cached"] is True
# is_cached should be [True] when retrieved from cache
assert records[0]["is_cached"] == [True]
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
@with_feature_flags(GLOBAL_ASYNC_QUERIES=True)