mirror of
https://github.com/apache/superset.git
synced 2026-09-02 13:31:33 +00:00
91 lines
3.7 KiB
Python
91 lines
3.7 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
from typing import Optional, Union
|
|
|
|
import pandas as pd
|
|
from flask_babel import gettext as _
|
|
|
|
from superset.exceptions import InvalidPostProcessingError
|
|
from superset.utils.pandas_postprocessing.utils import RESAMPLE_METHOD
|
|
|
|
# Upper bound on the number of rows a resample may project. ``rule`` arrives
|
|
# through the post-processing ``options`` dict, which is not schema-validated;
|
|
# without a cap, upsampling a multi-day span to e.g. ``1ns`` projects ~1e14
|
|
# rows from a single request.
|
|
MAX_RESAMPLE_ROWS = 1_000_000
|
|
|
|
|
|
def resample(
|
|
df: pd.DataFrame,
|
|
rule: str,
|
|
method: str,
|
|
fill_value: Optional[Union[float, int]] = None,
|
|
) -> pd.DataFrame:
|
|
"""
|
|
support upsampling in resample
|
|
|
|
:param df: DataFrame to resample.
|
|
:param rule: The offset string representing target conversion.
|
|
:param method: How to fill the NaN value after resample.
|
|
:param fill_value: What values do fill missing.
|
|
:return: DataFrame after resample
|
|
:raises InvalidPostProcessingError: If the request in incorrect
|
|
"""
|
|
if not isinstance(df.index, pd.DatetimeIndex):
|
|
raise InvalidPostProcessingError(_("Resample operation requires DatetimeIndex"))
|
|
if method not in RESAMPLE_METHOD:
|
|
raise InvalidPostProcessingError(
|
|
_("Resample method should be in ") + ", ".join(RESAMPLE_METHOD) + "."
|
|
)
|
|
|
|
if len(df):
|
|
try:
|
|
step = pd.Timedelta(pd.tseries.frequencies.to_offset(rule))
|
|
except ValueError:
|
|
# Non-fixed frequencies (month, quarter, year) have no fixed
|
|
# Timedelta; their projected row count is bounded by the span in
|
|
# days and needs no cap. Invalid rules fail in ``df.resample``.
|
|
step = None
|
|
if step is not None and step.value > 0:
|
|
span = df.index.max() - df.index.min()
|
|
# pandas snaps the first resample bin to the nearest frequency
|
|
# multiple at or before the observed span (and may extend the
|
|
# last bin similarly), so the actual bin count can exceed a
|
|
# naive span/step projection by one. Add a margin so the check
|
|
# cannot under-count due to that alignment.
|
|
projected_rows = span.value // step.value + 2
|
|
if projected_rows > MAX_RESAMPLE_ROWS:
|
|
raise InvalidPostProcessingError(
|
|
_(
|
|
"Resample rule would project %(rows)s rows, "
|
|
"exceeding the limit of %(max)s rows",
|
|
rows=projected_rows,
|
|
max=MAX_RESAMPLE_ROWS,
|
|
)
|
|
)
|
|
|
|
if method == "asfreq" and fill_value is not None:
|
|
_df = df.resample(rule).asfreq(fill_value=fill_value)
|
|
_df = _df.fillna(fill_value)
|
|
elif method == "linear":
|
|
_df = df.resample(rule).interpolate()
|
|
else:
|
|
_df = getattr(df.resample(rule), method)()
|
|
if method in ("ffill", "bfill"):
|
|
_df = getattr(_df, method)()
|
|
return _df
|