feat: new Columnar upload form and API (#28192)

This commit is contained in:
Daniel Vaz Gaspar
2024-05-06 15:51:42 +01:00
committed by GitHub
parent f5843fe588
commit 9a339f08a7
29 changed files with 2267 additions and 1232 deletions

View File

@@ -24,6 +24,7 @@ from typing import Any
import pandas as pd
import pytest
from werkzeug.datastructures import FileStorage
@pytest.fixture
@@ -31,7 +32,9 @@ def dttm() -> datetime:
return datetime.strptime("2019-01-02 03:04:05.678900", "%Y-%m-%d %H:%M:%S.%f")
def create_csv_file(data: list[list[str]] | None = None, delimiter=",") -> BytesIO:
def create_csv_file(
data: list[list[str]] | None = None, delimiter=",", filename="test.csv"
) -> FileStorage:
data = (
[
["Name", "Age", "City"],
@@ -46,14 +49,27 @@ def create_csv_file(data: list[list[str]] | None = None, delimiter=",") -> Bytes
for row in data:
writer.writerow(row)
output.seek(0)
bytes_buffer = BytesIO(output.getvalue().encode("utf-8"))
return bytes_buffer
buffer = BytesIO(output.getvalue().encode("utf-8"))
return FileStorage(stream=buffer, filename=filename)
def create_excel_file(data: dict[str, list[Any]] | None = None) -> BytesIO:
def create_excel_file(
data: dict[str, list[Any]] | None = None, filename="test.xls"
) -> FileStorage:
data = {"Name": ["John"], "Age": [30], "City": ["New York"]} if not data else data
excel_buffer = BytesIO()
buffer = BytesIO()
df = pd.DataFrame(data)
df.to_excel(excel_buffer, index=False)
excel_buffer.seek(0)
return excel_buffer
df.to_excel(buffer, index=False)
buffer.seek(0)
return FileStorage(stream=buffer, filename=filename)
def create_columnar_file(
data: dict[str, list[Any]] | None = None, filename="test.parquet"
) -> FileStorage:
data = {"Name": ["John"], "Age": [30], "City": ["New York"]} if not data else data
buffer = BytesIO()
df = pd.DataFrame(data)
df.to_parquet(buffer, index=False)
buffer.seek(0)
return FileStorage(stream=buffer, filename=filename)