diff --git a/runtime/transformations/components/time-series-base-components/aggregate_time_series_in_moving_window_100_12bb2c56-2adc-4c5d-bdaf-9c39ea247cd7.py b/runtime/transformations/components/time-series-base-components/aggregate_time_series_in_moving_window_100_12bb2c56-2adc-4c5d-bdaf-9c39ea247cd7.py new file mode 100644 index 000000000..55573212f --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/aggregate_time_series_in_moving_window_100_12bb2c56-2adc-4c5d-bdaf-9c39ea247cd7.py @@ -0,0 +1,621 @@ +"""Documentation for Aggregate Time Series in Moving Window + +# Aggregate Time Series in Moving Window + +## Description +This component calculates aggregation values on fixed time windows that move +over the data in a regular rhythm. + +## Inputs +* **timeseries** (Pandas Series): Series to perform the moving window aggregation on. +* **aggregator** (String, default value: "mean"): Aggregation function to + apply in each window. Must be one of "mean", "median", "min", "max", or + "std". +* **min_periods** (Integer, default value: 1): Minimum number of valid values + in a window required to return an aggregation result. If fewer values are + present, the result is `NaN`. +* **window_size** (String, default value: "15min"): Time span covered by each + window. Can be either a Pandas frequency string based on [date offset + aliases](https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases) + or a timedelta string. +* **window_frequency** (String, default value: "5min"): Time span between two + consecutive window starts. For directly consecutive, non-overlapping windows + set **window_frequency** to the same value as **window_size**. +* **frequency_offset** (String, default value: "0min"): Optional shift of the + complete window rhythm. In most cases no shift is needed, so the default is + `"0min"`. +* **interval_type** (String, default value: "left_closed"): Controls whether + values exactly on a window edge belong to the window or not. + Supported values are "left_closed", "right_open", "right_closed", + "left_open", "closed", and "open". + In practice: + - `left_closed` / `right_open`: left edge included, right edge excluded + - `right_closed` / `left_open`: right edge included, left edge excluded + - `closed`: both edges included + - `open`: both edges excluded +* **label_position** (String, default value: "left"): Determines which + timestamp represents each window in the result. Must be one of "left", + "center", or "right". For non-fixed calendar windows such as monthly or + quarterly windows, `center` is not supported. + +## Outputs +* **window_values** (Pandas Series): Series with the calculated aggregation values of each window. + +## Details +1. The component checks the configuration and maps `interval_type` to the internal window boundary mode. +2. It converts the time settings (`window_size`, `window_frequency`, `frequency_offset`) to pandas time objects. +3. The input series is sorted by timestamp to ensure a stable and deterministic calculation order. +4. A regular window rhythm is built from `window_frequency` and + `frequency_offset`. +5. If the settings allow it, a fast `resample` path is used. Otherwise, a + general `rolling` path is used. +6. For each window, the selected `aggregator` is calculated. +7. If a window has fewer than `min_periods` valid values, its aggregation + result is set to `NaN`. +8. The output labels are placed at the `left`, `center`, or `right` of the + window according to `label_position`. +9. The component returns the aggregated series (`window_values`). + +## Examples +The json input of a typical call of this component is +``` +{ + "timeseries": { + "2025-12-06 23:17:14+00:00": 14.360453, + "2025-12-06 23:18:14+00:00": 14.872439, + "2025-12-06 23:19:14+00:00": 15.803046, + "2025-12-06 23:20:14+00:00": 13.661063, + "2025-12-06 23:21:14+00:00": 14.969653, + "2025-12-06 23:22:14+00:00": 13.530460, + "2025-12-06 23:23:14+00:00": 14.009814, + "2025-12-06 23:24:14+00:00": 13.708907, + "2025-12-06 23:25:14+00:00": 15.119281, + "2025-12-06 23:26:14+00:00": 15.666053, + "2025-12-06 23:27:14+00:00": 15.488811, + "2025-12-06 23:28:14+00:00": 15.515572, + "2025-12-06 23:29:14+00:00": 14.767891, + "2025-12-06 23:30:14+00:00": 13.726783, + "2025-12-06 23:31:14+00:00": 14.997352, + "2025-12-06 23:32:14+00:00": 13.015161, + "2025-12-06 23:33:14+00:00": 15.079884, + "2025-12-06 23:34:14+00:00": 15.001638, + "2025-12-06 23:35:14+00:00": 16.035476, + "2025-12-06 23:36:14+00:00": 14.545874, + "2025-12-06 23:37:14+00:00": 13.970260, + "2025-12-06 23:38:14+00:00": 15.045999, + "2025-12-06 23:39:14+00:00": 13.824304, + "2025-12-06 23:40:14+00:00": 13.620449, + "2025-12-06 23:41:14+00:00": 15.146005, + "2025-12-06 23:42:14+00:00": 14.796941, + "2025-12-06 23:43:14+00:00": 17.036046, + "2025-12-06 23:44:14+00:00": 14.976151, + "2025-12-06 23:45:14+00:00": 14.637615, + "2025-12-06 23:46:14+00:00": 15.195011, + "2025-12-06 23:47:14+00:00": 14.082022, + "2025-12-06 23:48:14+00:00": 15.311653, + "2025-12-06 23:49:14+00:00": 15.534084, + "2025-12-06 23:50:14+00:00": 15.020099, + "2025-12-06 23:51:14+00:00": 14.020416, + "2025-12-06 23:52:14+00:00": 13.779699, + "2025-12-06 23:53:14+00:00": 14.407253, + "2025-12-06 23:54:14+00:00": 15.180839, + "2025-12-06 23:55:14+00:00": 14.618573, + "2025-12-06 23:56:14+00:00": 14.194774, + "2025-12-06 23:57:14+00:00": 14.653221, + "2025-12-06 23:58:14+00:00": 14.911146, + "2025-12-06 23:59:14+00:00": 14.951289, + "2025-12-07 00:00:14+00:00": 15.005731, + "2025-12-07 00:01:14+00:00": 15.708401, + "2025-12-07 00:02:14+00:00": 13.605439, + "2025-12-07 00:03:14+00:00": 14.620927, + "2025-12-07 00:04:14+00:00": 15.181250, + "2025-12-07 00:05:14+00:00": 13.912724, + "2025-12-07 00:06:14+00:00": 14.060204 + }, + "window_size": "15min", + "window_frequency": "5min", + "frequency_offset": "0min", + "interval_type": "left_closed", + "aggregator": "mean", + "min_periods": 1, + "label_position": "left" +} +``` +The expected output is +``` +"window_values": { + "2025-12-06T23:05:00.000Z": 15.0119793333, + "2025-12-06T23:10:00.000Z": 14.364479375, + "2025-12-06T23:15:00.000Z": 14.7287263846, + "2025-12-06T23:20:00.000Z": 14.5505548667, + "2025-12-06T23:25:00.000Z": 14.7866892667, + "2025-12-06T23:30:00.000Z": 14.7212215333, + "2025-12-06T23:35:00.000Z": 14.9171926667, + "2025-12-06T23:40:00.000Z": 14.8496188667, + "2025-12-06T23:45:00.000Z": 14.6998462667, + "2025-12-06T23:50:00.000Z": 14.6572704667, + "2025-12-06T23:55:00.000Z": 14.6186399167, + "2025-12-07T00:00:00.000Z": 14.5849537143, + "2025-12-07T00:05:00.000Z": 13.986464 +}, +``` +""" + +from typing import Literal + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + + +def freqstr2dateoffset(freqstr: str) -> pd.DateOffset: + """Transform frequency string to Pandas DateOffset.""" + return pd.tseries.frequencies.to_offset(freqstr) + + +def freqstr2timedelta(freqstr: str) -> pd.Timedelta: + """Transform frequency string to Pandas Timedelta.""" + try: + return pd.to_timedelta(freqstr) + except ValueError: + return pd.to_timedelta(freqstr2dateoffset(freqstr)) + + +def is_fixed_frequency(offset: pd.DateOffset) -> bool: + """Return True if a pandas offset represents a fixed duration.""" + try: + _ = offset.nanos + return True + except ValueError: + return False + + +def shift_timestamp_to_the_left_onto_rhythm( + timestamp: pd.Timestamp, + window_frequency: pd.DateOffset, + frequency_offset: pd.Timedelta, +) -> pd.Timestamp: + """Shift a timestamp to the left in the rhythm. + + The parameters window_frequency and frequency_offset define a kind of "rhythm". + For example a window_frequency of "5min" and a frequency_offset of "1min" define the + rhythm which contains all timestamps, where the minutes are 01, 06, 11, 16, and so on. + The provided timestamp is shifted to the left onto the closest timestamp of this rhythm. + + Conveniently, the Pandas class Timestamp comes with a method `floor`, which is similar to + the mathematical method `floor`, but instead of a decimal place takes into account the + specified frequency. + + It is not completely obvious how the frequency_offset needs to be taken into account so that the + shifted timestamp actually lies in the desired interval: + timestamp - window_frequency < shifted <= timestamp + so in the following a little proof is provided: + + On the one hand we have: + frequency_offset < window_frequency + shifted = (timestamp - frequency_offset).floor(freq=window_frequency) + frequency_offset + <-> shifted - frequency_offset = (timestamp - frequency_offset).floor(freq=window_frequency) + <= timestamp - frequency_offset + -> shifted <= timestamp + + On the other hand: + shifted - frequency_offset = (timestamp - frequency_offset).floor(freq=window_frequency) + > timestamp - frequency_offset - window_frequency + -> shifted > timestamp - window_frequency + """ + return (timestamp - frequency_offset).floor(freq=window_frequency) + frequency_offset + + +def shift_timestamp_to_the_right_onto_rhythm( + timestamp: pd.Timestamp, + window_frequency: pd.DateOffset, + frequency_offset: pd.Timedelta, +) -> pd.Timestamp: + """Shift a timestamp to the right in the rhythm. + + The parameters window_frequency and frequency_offset define a kind of "rhythm". + The specified timestamp is shifted to the right onto the closest timestamp of this rhythm. + + Conveniently, the Pandas class Timestamp has a method `ceil` that is similar to the + mathematical method `ceil`, but instead of a decimal place, it takes into account the + specified frequency. + + It is not completely obvious how the frequency_offset must be taken into account so that + the shifted timestamp actually lies in the desired interval: + timestamp <= shifted < timestamp + window_frequency + The proof that the implemented code fulfills this requirement is analogous to the one for + `shift_timestamp_to_the_left_in_rhythm`. + """ + return (timestamp - frequency_offset).ceil(freq=window_frequency) + frequency_offset + + +def right_window_edge_from_left_window_edge( + left_window_edge: pd.Timestamp, window_size: pd.DateOffset +) -> pd.Timestamp: + return left_window_edge + window_size + + +def determine_right_window_edges( + first_index: pd.Timestamp, + last_index: pd.Timestamp, + window_size: pd.DateOffset, + window_frequency: pd.DateOffset, + frequency_offset: pd.Timedelta, +) -> pd.DatetimeIndex: + """Determine right window edges of all windows containing first_index and last_index. + + This function determines the right edges of the windows for which the mean shall be calculated. + + The parameters window_frequency and frequency_offset define an endless "rhythm" where left + edges of windows should be located. From first_index and last_index the left edges of earliest + and latest window containing these timestamps are determined. Using the window_size left window + edges are transformed to right window edges. Finally the list of all right window edges is + determined and returned. + + The right edges are determined because the Pandas rolling method only offers the possibility to + operate on windows that center around the current point or on windows for which the current + point is the right edge. + """ + earliest_possible_left_edge_of_window_containing_first_index = first_index - window_size + first_window_left_edge = shift_timestamp_to_the_right_onto_rhythm( + timestamp=earliest_possible_left_edge_of_window_containing_first_index, + window_frequency=window_frequency, + frequency_offset=frequency_offset, + ) + first_window_right_boundary = right_window_edge_from_left_window_edge( + left_window_edge=first_window_left_edge, window_size=window_size + ) + + latest_possible_left_edge_of_window_containing_last_index = last_index + last_window_left_boundary = shift_timestamp_to_the_left_onto_rhythm( + timestamp=latest_possible_left_edge_of_window_containing_last_index, + window_frequency=window_frequency, + frequency_offset=frequency_offset, + ) + last_window_right_boundary = right_window_edge_from_left_window_edge( + left_window_edge=last_window_left_boundary, window_size=window_size + ) + + return pd.date_range( + start=first_window_right_boundary, + end=last_window_right_boundary, + freq=window_frequency, + inclusive="both", + ) + + +def calculate_moving_time_window( + timeseries: pd.Series, + window_size: pd.DateOffset, + window_frequency: pd.DateOffset, + frequency_offset: pd.Timedelta, + inclusive: Literal["left", "right", "both", "neither"], + label_position: Literal["left", "center", "right"], + aggregator: Literal["mean", "median", "min", "max", "std"], +) -> tuple[pd.Series, pd.Series]: + """Calculate periodically shifting window aggregates for a constant time window. + + timeseries (Pandas Series): Series to perform the periodically shifting time window + calculation on. + window_size (Pandas DateOffset): Time span of each window. + window_frequency (Pandas DateOffset): Frequency of windows for which the aggregation is calculated, + i.e. time delta between the start (or end) of each two consecutive windows. + For directly consecutive, non-overlapping windows set window_frequency to the same value as + window_size. If the window_frequency is smaller than the window_size the windows will + overlap. If the window_frequency is larger than the window_size, there will be gaps between + each two successive windows. + frequency_offset (Pandas Timedelta): Offset of the window starts compared to + 1970-01-01 00:00:00. In most cases no offset is necessary, so this can be set to zero, + i.e. "0". + inclusive (string): One of "left", "right", "both", or "neither". + In case a datapoint is on the left or right border of a window this option + determines if it belongs to that window or not, but potentially a neighbouring window. + label_position (string): The string must be either "left", "center", or "right". + This option determines which timestamp is provided to represent the window for the + corresponding aggregation in the output time series. + + To reduce the runtime, the Pandas function resample is used when possible + (window_frequency and window_size are identical and inclusive is "left" or "right"). + Otherwise, the Pandas function rolling is used. + """ + timeseries = timeseries.sort_index() + + frequency_offset = frequency_offset % window_frequency + + if window_size == window_frequency and inclusive in ["left", "right"]: + # resample is the fastest method, if it can be used + resampled = timeseries.resample( + rule=window_size, + closed=inclusive, + label="right", + origin="epoch", + offset=frequency_offset, + ) + result = getattr(resampled, aggregator)() + counts = resampled.count() + else: + # default label position in rolling is right + # left is not possible, only alternative is center + right_window_boundaries = determine_right_window_edges( + first_index=timeseries.index[0], + last_index=timeseries.index[-1], + window_size=window_size, + window_frequency=window_frequency, + frequency_offset=frequency_offset, + ) + + reindexed_timeseries = timeseries.reindex( + index=right_window_boundaries.union(timeseries.index) + ) + + rolling_obj = reindexed_timeseries.rolling(window_size.freqstr, closed=inclusive) + result = getattr(rolling_obj, aggregator)() + counts = rolling_obj.count() + + result = result.reindex(index=right_window_boundaries) + counts = counts.reindex(index=right_window_boundaries) + + if label_position == "center": + result = result.shift(freq=-pd.to_timedelta(window_size) / 2) + counts = counts.shift(freq=-pd.to_timedelta(window_size) / 2) + elif label_position == "left": + result = result.shift(freq=-pd.to_timedelta(window_size)) + counts = counts.shift(freq=-pd.to_timedelta(window_size)) + + return result, counts + + +def validate_inputs( + timeseries: pd.Series, + aggregator: str, + min_periods, + window_size: str, + window_frequency: str, + frequency_offset: str, + interval_type: str, + label_position: str, +) -> tuple[int, pd.DateOffset, pd.DateOffset, pd.Timedelta]: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series.", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not isinstance(timeseries.index, pd.DatetimeIndex): + raise ComponentInputValidationException( + "timeseries index must be a pandas DatetimeIndex.", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty.", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric.", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + finite_mask = np.isfinite(timeseries.dropna().to_numpy(dtype=float)) + if not finite_mask.all(): + raise ComponentInputValidationException( + "timeseries values must be finite numbers.", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + allowed_aggregators = {"mean", "median", "min", "max", "std"} + if aggregator not in allowed_aggregators: + raise ComponentInputValidationException( + "aggregator must be one of: mean, median, min, max, std", + error_code="422", + invalid_component_inputs=["aggregator"], + ) + + allowed_interval_types = { + "closed", + "open", + "left_closed", + "right_open", + "right_closed", + "left_open", + } + if interval_type not in allowed_interval_types: + raise ComponentInputValidationException( + "interval_type must be one of: closed, open, left_closed, right_open, right_closed, left_open", + error_code="422", + invalid_component_inputs=["interval_type"], + ) + + if label_position not in {"left", "center", "right"}: + raise ComponentInputValidationException( + "label_position must be one of: left, center, right", + error_code="422", + invalid_component_inputs=["label_position"], + ) + + if isinstance(min_periods, str): + min_periods = min_periods.strip() + if not min_periods.isdigit(): + raise ComponentInputValidationException( + "min_periods must be an integer >= 1.", + error_code="422", + invalid_component_inputs=["min_periods"], + ) + min_periods = int(min_periods) + if not isinstance(min_periods, int) or min_periods < 1: + raise ComponentInputValidationException( + "min_periods must be an integer >= 1.", + error_code="422", + invalid_component_inputs=["min_periods"], + ) + + try: + window_size_offset = freqstr2dateoffset(window_size) + except Exception as exc: + raise ComponentInputValidationException( + "window_size must be a valid positive frequency string.", + error_code="422", + invalid_component_inputs=["window_size"], + ) from exc + + try: + window_frequency_offset = freqstr2dateoffset(window_frequency) + except Exception as exc: + raise ComponentInputValidationException( + "window_frequency must be a valid positive frequency string.", + error_code="422", + invalid_component_inputs=["window_frequency"], + ) from exc + + try: + frequency_offset_delta = freqstr2timedelta(frequency_offset) + except Exception as exc: + raise ComponentInputValidationException( + "frequency_offset must be a valid timedelta/frequency string.", + error_code="422", + invalid_component_inputs=["frequency_offset"], + ) from exc + + reference_ts = pd.Timestamp("1970-01-01T00:00:00Z") + if not (reference_ts + window_size_offset > reference_ts): + raise ComponentInputValidationException( + "window_size must represent a positive duration.", + error_code="422", + invalid_component_inputs=["window_size"], + ) + if not (reference_ts + window_frequency_offset > reference_ts): + raise ComponentInputValidationException( + "window_frequency must represent a positive duration.", + error_code="422", + invalid_component_inputs=["window_frequency"], + ) + + if label_position == "center" and not is_fixed_frequency(window_size_offset): + raise ComponentInputValidationException( + "label_position='center' is only supported for fixed-duration windows such as '15min', '1h', or '7D'.", + error_code="422", + invalid_component_inputs=["label_position", "window_size"], + ) + + return ( + min_periods, + window_size_offset, + window_frequency_offset, + frequency_offset_delta, + ) + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "aggregator": {"data_type": "STRING", "default_value": "mean"}, + "min_periods": {"data_type": "INT", "default_value": 1}, + "window_size": {"data_type": "STRING", "default_value": "15min"}, + "window_frequency": {"data_type": "STRING", "default_value": "5min"}, + "frequency_offset": {"data_type": "STRING", "default_value": "0min"}, + "interval_type": {"data_type": "STRING", "default_value": "left_closed"}, + "label_position": {"data_type": "STRING", "default_value": "left"}, + }, + "outputs": { + "window_values": {"data_type": "SERIES"}, + }, + "name": "Aggregate Time Series in Moving Window", + "category": "Time Series Base Components", + "description": "Calculate moving time window aggregation values.", + "version_tag": "1.0.0", + "id": "12bb2c56-2adc-4c5d-bdaf-9c39ea247cd7", + "revision_group_id": "f1642f90-cc86-48f9-8feb-50e750204d40", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + aggregator="mean", + min_periods=1, + window_size="15min", + window_frequency="5min", + frequency_offset="0min", + interval_type="left_closed", + label_position="left", +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Map the user-facing interval type to the internal pandas mode. + inclusive_string_from_interval_type = { + "closed": "both", + "open": "neither", + "left_closed": "left", + "right_open": "left", + "right_closed": "right", + "left_open": "right", + } + + # Step 2: Validate inputs and convert time settings to pandas objects. + ( + min_periods, + window_size_offset, + window_frequency_offset, + frequency_offset_delta, + ) = validate_inputs( + timeseries=timeseries, + aggregator=aggregator, + min_periods=min_periods, + window_size=window_size, + window_frequency=window_frequency, + frequency_offset=frequency_offset, + interval_type=interval_type, + label_position=label_position, + ) + + # Step 3: Calculate window aggregates and valid-value counts. + window_values, valid_value_counts = calculate_moving_time_window( + timeseries=timeseries, + window_size=window_size_offset, + window_frequency=window_frequency_offset, + frequency_offset=frequency_offset_delta, + inclusive=inclusive_string_from_interval_type[interval_type], + label_position=label_position, + aggregator=aggregator, + ) + + # Step 4: Apply the minimum valid-value requirement per window. + window_values = window_values.where(valid_value_counts >= min_periods) + + # Step 5: Return the aggregated output series. + return { + "window_values": window_values, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2025-12-06 23:17:14+00:00": 14.360453,\n "2025-12-06 23:18:14+00:00": 14.872439,\n "2025-12-06 23:19:14+00:00": 15.803046,\n "2025-12-06 23:20:14+00:00": 13.661063,\n "2025-12-06 23:21:14+00:00": 14.969653,\n "2025-12-06 23:22:14+00:00": 13.530460,\n "2025-12-06 23:23:14+00:00": 14.009814,\n "2025-12-06 23:24:14+00:00": 13.708907,\n "2025-12-06 23:25:14+00:00": 15.119281,\n "2025-12-06 23:26:14+00:00": 15.666053,\n "2025-12-06 23:27:14+00:00": 15.488811,\n "2025-12-06 23:28:14+00:00": 15.515572,\n "2025-12-06 23:29:14+00:00": 14.767891,\n "2025-12-06 23:30:14+00:00": 13.726783,\n "2025-12-06 23:31:14+00:00": 14.997352,\n "2025-12-06 23:32:14+00:00": 13.015161,\n "2025-12-06 23:33:14+00:00": 15.079884,\n "2025-12-06 23:34:14+00:00": 15.001638,\n "2025-12-06 23:35:14+00:00": 16.035476,\n "2025-12-06 23:36:14+00:00": 14.545874,\n "2025-12-06 23:37:14+00:00": 13.970260,\n "2025-12-06 23:38:14+00:00": 15.045999,\n "2025-12-06 23:39:14+00:00": 13.824304,\n "2025-12-06 23:40:14+00:00": 13.620449,\n "2025-12-06 23:41:14+00:00": 15.146005,\n "2025-12-06 23:42:14+00:00": 14.796941,\n "2025-12-06 23:43:14+00:00": 17.036046,\n "2025-12-06 23:44:14+00:00": 14.976151,\n "2025-12-06 23:45:14+00:00": 14.637615,\n "2025-12-06 23:46:14+00:00": 15.195011,\n "2025-12-06 23:47:14+00:00": 14.082022,\n "2025-12-06 23:48:14+00:00": 15.311653,\n "2025-12-06 23:49:14+00:00": 15.534084,\n "2025-12-06 23:50:14+00:00": 15.020099,\n "2025-12-06 23:51:14+00:00": 14.020416,\n "2025-12-06 23:52:14+00:00": 13.779699,\n "2025-12-06 23:53:14+00:00": 14.407253,\n "2025-12-06 23:54:14+00:00": 15.180839,\n "2025-12-06 23:55:14+00:00": 14.618573,\n "2025-12-06 23:56:14+00:00": 14.194774,\n "2025-12-06 23:57:14+00:00": 14.653221,\n "2025-12-06 23:58:14+00:00": 14.911146,\n "2025-12-06 23:59:14+00:00": 14.951289,\n "2025-12-07 00:00:14+00:00": 15.005731,\n "2025-12-07 00:01:14+00:00": 15.708401,\n "2025-12-07 00:02:14+00:00": 13.605439,\n "2025-12-07 00:03:14+00:00": 14.620927,\n "2025-12-07 00:04:14+00:00": 15.181250,\n "2025-12-07 00:05:14+00:00": 13.912724,\n "2025-12-07 00:06:14+00:00": 14.060204\n}' + }, + } + ] +} +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2025-12-06 23:17:14+00:00": 14.360453,\n "2025-12-06 23:18:14+00:00": 14.872439,\n "2025-12-06 23:19:14+00:00": 15.803046,\n "2025-12-06 23:20:14+00:00": 13.661063,\n "2025-12-06 23:21:14+00:00": 14.969653,\n "2025-12-06 23:22:14+00:00": 13.530460,\n "2025-12-06 23:23:14+00:00": 14.009814,\n "2025-12-06 23:24:14+00:00": 13.708907,\n "2025-12-06 23:25:14+00:00": 15.119281,\n "2025-12-06 23:26:14+00:00": 15.666053,\n "2025-12-06 23:27:14+00:00": 15.488811,\n "2025-12-06 23:28:14+00:00": 15.515572,\n "2025-12-06 23:29:14+00:00": 14.767891,\n "2025-12-06 23:30:14+00:00": 13.726783,\n "2025-12-06 23:31:14+00:00": 14.997352,\n "2025-12-06 23:32:14+00:00": 13.015161,\n "2025-12-06 23:33:14+00:00": 15.079884,\n "2025-12-06 23:34:14+00:00": 15.001638,\n "2025-12-06 23:35:14+00:00": 16.035476,\n "2025-12-06 23:36:14+00:00": 14.545874,\n "2025-12-06 23:37:14+00:00": 13.970260,\n "2025-12-06 23:38:14+00:00": 15.045999,\n "2025-12-06 23:39:14+00:00": 13.824304,\n "2025-12-06 23:40:14+00:00": 13.620449,\n "2025-12-06 23:41:14+00:00": 15.146005,\n "2025-12-06 23:42:14+00:00": 14.796941,\n "2025-12-06 23:43:14+00:00": 17.036046,\n "2025-12-06 23:44:14+00:00": 14.976151,\n "2025-12-06 23:45:14+00:00": 14.637615,\n "2025-12-06 23:46:14+00:00": 15.195011,\n "2025-12-06 23:47:14+00:00": 14.082022,\n "2025-12-06 23:48:14+00:00": 15.311653,\n "2025-12-06 23:49:14+00:00": 15.534084,\n "2025-12-06 23:50:14+00:00": 15.020099,\n "2025-12-06 23:51:14+00:00": 14.020416,\n "2025-12-06 23:52:14+00:00": 13.779699,\n "2025-12-06 23:53:14+00:00": 14.407253,\n "2025-12-06 23:54:14+00:00": 15.180839,\n "2025-12-06 23:55:14+00:00": 14.618573,\n "2025-12-06 23:56:14+00:00": 14.194774,\n "2025-12-06 23:57:14+00:00": 14.653221,\n "2025-12-06 23:58:14+00:00": 14.911146,\n "2025-12-06 23:59:14+00:00": 14.951289,\n "2025-12-07 00:00:14+00:00": 15.005731,\n "2025-12-07 00:01:14+00:00": 15.708401,\n "2025-12-07 00:02:14+00:00": 13.605439,\n "2025-12-07 00:03:14+00:00": 14.620927,\n "2025-12-07 00:04:14+00:00": 15.181250,\n "2025-12-07 00:05:14+00:00": 13.912724,\n "2025-12-07 00:06:14+00:00": 14.060204\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/align_and_merge_two_time_series_100_79ffe3ff-346f-4dad-ab38-ecb7b477325d.py b/runtime/transformations/components/time-series-base-components/align_and_merge_two_time_series_100_79ffe3ff-346f-4dad-ab38-ecb7b477325d.py new file mode 100644 index 000000000..c3da56cdb --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/align_and_merge_two_time_series_100_79ffe3ff-346f-4dad-ab38-ecb7b477325d.py @@ -0,0 +1,466 @@ +"""Documentation for Align and Merge Two Time Series + +# Align and Merge Two Time Series + +## Description +Component to align two separate time series on a datetime axis and combine them +into one result series. It supports exact matching, nearest-neighbor matching, and +forward-fill matching. + +## Inputs +- **timeseries_1** (Pandas Series): + The first input time series. The index must contain timestamps, and the + values must be numeric. +- **timeseries_2** (Pandas Series): + The second input time series. The index must contain timestamps, and the + values must be numeric. +- **align_strategy** (String, default value: "exact"): + Controls how the non-reference series is aligned to the reference index. + For `join_type="left"`, this usually means aligning `timeseries_2` to the + index of `timeseries_1`. For `join_type="right"`, it means aligning + `timeseries_1` to the index of `timeseries_2`. + - `exact`: only identical timestamps are matched. + - `nearest`: the temporally nearest value is matched. + - `forward_fill`: the latest earlier value is matched. +- **join_type** (String, default value: "inner"): + Controls which time index is used for the result. + - `left`: use the index of `timeseries_1`. + - `right`: use the index of `timeseries_2`. + - `inner`: for `align_strategy="exact"`, use the exact timestamp + intersection of both series. For non-exact strategies, use the index of + `timeseries_1` as the reference index and keep only timestamps where the + other series could be aligned successfully. +- **tolerance** (String, default value: null): + Optional maximum allowed time distance for `nearest` or `forward_fill`. + If the aligned value is farther away than this tolerance, no match is + created. Use a fixed timedelta string such as `30s`, `5min`, or `1h`, or + `null` to disable this check. For `align_strategy="exact"`, this input has + no effect. +- **missing_policy** (String, default value: "keep_nan"): + Controls what happens after alignment if one side is missing. + - `keep_nan`: keep the timestamp and let the merge result become `NaN` for + operations that need both sides. + - `drop_if_any_missing`: drop timestamps where either aligned side is + missing before applying the merge operation. +- **merge_operation** (String, default value: "sum"): + Defines how the two aligned values are merged into one result. + - `left`: use the aligned value from `timeseries_1`. + - `right`: use the aligned value from `timeseries_2`. + - `sum`: add both aligned values. + - `mean`: average both aligned values. + - `min`: take the minimum of both aligned values. + - `max`: take the maximum of both aligned values. + - `difference`: calculate `timeseries_1 - timeseries_2`. + +## Outputs +- **combined_series** (Pandas Series): + The combined result series after alignment and merge. + +## Details +1. Both input series are sorted by time and duplicate timestamps are merged by mean. +2. Depending on `join_type`, either `timeseries_1`, `timeseries_2`, or an + exact timestamp intersection is used as the reference index. +3. Depending on `align_strategy`, the non-reference series is aligned by exact + matching, nearest-neighbor matching, or forward-fill matching. + For `join_type="right"`, this means that `timeseries_1` is aligned to the + index of `timeseries_2`. +4. If `tolerance` is set, matches that are farther away than this threshold are + rejected for `nearest` and `forward_fill`. + For `align_strategy="exact"`, `tolerance` is ignored. +5. After alignment, `missing_policy` decides whether timestamps with missing + aligned values are kept or dropped. +6. Finally, `merge_operation` combines the two aligned values into one result + value per timestamp. +7. If both series are strongly irregular or visibly offset in time, it is often + better to first bring them onto a shared grid with `resample_time_series` + and only then use this component. +8. The `inner` mode is symmetric only for `align_strategy="exact"`. For + non-exact strategies, `inner` uses `timeseries_1` as the reference index + and then keeps only timestamps with successful matches from the other side. + +## Example +```json +{ + "timeseries_1": { + "2026-03-01T10:00:00Z": 100.0, + "2026-03-01T10:05:00Z": 105.0, + "2026-03-01T10:10:00Z": 110.0 + }, + "timeseries_2": { + "2026-03-01T10:00:00Z": 20.0, + "2026-03-01T10:05:00Z": 22.0, + "2026-03-01T10:10:00Z": 24.0 + }, + "align_strategy": "exact", + "join_type": "inner", + "tolerance": null, + "missing_policy": "keep_nan", + "merge_operation": "sum" +} +``` + +Expected output: +```json +{ + "combined_series": { + "2026-03-01T10:00:00Z": 120.0, + "2026-03-01T10:05:00Z": 127.0, + "2026-03-01T10:10:00Z": 134.0 + } +} +``` + +Second example with `nearest`: +```json +{ + "timeseries_1": { + "2026-03-01T10:00:00Z": 100.0, + "2026-03-01T10:05:00Z": 105.0, + "2026-03-01T10:10:00Z": 110.0 + }, + "timeseries_2": { + "2026-03-01T10:00:20Z": 20.0, + "2026-03-01T10:05:15Z": 22.0, + "2026-03-01T10:11:00Z": 24.0 + }, + "align_strategy": "nearest", + "join_type": "left", + "tolerance": "30s", + "missing_policy": "drop_if_any_missing", + "merge_operation": "sum" +} +``` + +Expected output: +```json +{ + "combined_series": { + "2026-03-01T10:00:00Z": 120.0, + "2026-03-01T10:05:00Z": 127.0 + } +} +``` +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + +ALIGN_STRATEGIES = {"exact", "nearest", "forward_fill"} +JOIN_TYPES = {"inner", "left", "right"} +MISSING_POLICIES = {"keep_nan", "drop_if_any_missing"} +MERGE_OPERATIONS = {"left", "right", "sum", "mean", "min", "max", "difference"} + + +def parse_fixed_timedelta_string(value: str, input_name: str) -> pd.Timedelta: + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a valid fixed timedelta string like '30s', '5min', or '1h'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + try: + nanos = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '30s', '5min', or '1h'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + return pd.to_timedelta(nanos, unit="ns") + + +def validate_series(timeseries: pd.Series, input_name: str) -> None: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + f"{input_name} must be a pandas Series", + error_code="422", + invalid_component_inputs=[input_name], + ) + if timeseries.empty: + raise ComponentInputValidationException( + f"{input_name} must not be empty", + error_code="422", + invalid_component_inputs=[input_name], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + f"{input_name} index must be datetime", + error_code="422", + invalid_component_inputs=[input_name], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + f"{input_name} values must be numeric", + error_code="422", + invalid_component_inputs=[input_name], + ) + if not np.isfinite(timeseries.dropna().to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + f"{input_name} must not contain inf or -inf values", + error_code="422", + invalid_component_inputs=[input_name], + ) + if timeseries.dropna().empty: + raise ComponentInputValidationException( + f"{input_name} must contain at least one non-missing numeric value", + error_code="422", + invalid_component_inputs=[input_name], + ) + + +def validate_inputs( + timeseries_1: pd.Series, + timeseries_2: pd.Series, + align_strategy: str, + join_type: str, + tolerance: str | None, + missing_policy: str, + merge_operation: str, +) -> pd.Timedelta | None: + validate_series(timeseries_1, "timeseries_1") + validate_series(timeseries_2, "timeseries_2") + + if align_strategy not in ALIGN_STRATEGIES: + raise ComponentInputValidationException( + f"align_strategy must be one of {sorted(ALIGN_STRATEGIES)}", + error_code="422", + invalid_component_inputs=["align_strategy"], + ) + if join_type not in JOIN_TYPES: + raise ComponentInputValidationException( + f"join_type must be one of {sorted(JOIN_TYPES)}", + error_code="422", + invalid_component_inputs=["join_type"], + ) + if missing_policy not in MISSING_POLICIES: + raise ComponentInputValidationException( + f"missing_policy must be one of {sorted(MISSING_POLICIES)}", + error_code="422", + invalid_component_inputs=["missing_policy"], + ) + if merge_operation not in MERGE_OPERATIONS: + raise ComponentInputValidationException( + f"merge_operation must be one of {sorted(MERGE_OPERATIONS)}", + error_code="422", + invalid_component_inputs=["merge_operation"], + ) + + parsed_tolerance = None + if tolerance is not None: + parsed_tolerance = parse_fixed_timedelta_string(tolerance, "tolerance") + if parsed_tolerance <= pd.Timedelta(0): + raise ComponentInputValidationException( + "tolerance must be greater than zero", + error_code="422", + invalid_component_inputs=["tolerance"], + ) + + return parsed_tolerance + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared.astype(float) + + +def align_series_to_reference( + reference_index: pd.DatetimeIndex, + source_series: pd.Series, + align_strategy: str, + tolerance: pd.Timedelta | None, +) -> pd.Series: + if align_strategy == "exact": + return source_series.reindex(reference_index) + + reference_frame = pd.DataFrame({"timestamp": reference_index}) + source_frame = source_series.rename("value").reset_index() + source_frame.columns = ["timestamp", "value"] + + direction = "nearest" if align_strategy == "nearest" else "backward" + merged = pd.merge_asof( + reference_frame, + source_frame, + on="timestamp", + direction=direction, + tolerance=tolerance, + ) + return pd.Series(merged["value"].to_numpy(dtype=float), index=reference_index, dtype=float) + + +def build_aligned_frame( + series_1: pd.Series, + series_2: pd.Series, + align_strategy: str, + join_type: str, + tolerance: pd.Timedelta | None, +) -> pd.DataFrame: + if join_type == "right": + reference_index = series_2.index + aligned_1 = align_series_to_reference(reference_index, series_1, align_strategy, tolerance) + aligned_2 = series_2.reindex(reference_index) + elif join_type == "left": + reference_index = series_1.index + aligned_1 = series_1.reindex(reference_index) + aligned_2 = align_series_to_reference(reference_index, series_2, align_strategy, tolerance) + elif align_strategy == "exact": + reference_index = series_1.index.intersection(series_2.index) + aligned_1 = series_1.reindex(reference_index) + aligned_2 = series_2.reindex(reference_index) + else: + reference_index = series_1.index + aligned_1 = series_1.reindex(reference_index) + aligned_2 = align_series_to_reference(reference_index, series_2, align_strategy, tolerance) + matched = aligned_1.notna() & aligned_2.notna() + aligned_1 = aligned_1[matched] + aligned_2 = aligned_2[matched] + + return pd.DataFrame({"value_1": aligned_1, "value_2": aligned_2}) + + +def apply_missing_policy(aligned: pd.DataFrame, missing_policy: str) -> pd.DataFrame: + if missing_policy == "drop_if_any_missing": + return aligned.dropna(how="any") + return aligned + + +def merge_aligned_values(aligned: pd.DataFrame, merge_operation: str) -> pd.Series: + value_1 = aligned["value_1"] + value_2 = aligned["value_2"] + + if merge_operation == "left": + result = value_1 + elif merge_operation == "right": + result = value_2 + elif merge_operation == "sum": + result = value_1 + value_2 + elif merge_operation == "mean": + result = (value_1 + value_2) / 2.0 + elif merge_operation == "min": + result = aligned[["value_1", "value_2"]].min(axis=1, skipna=False) + elif merge_operation == "max": + result = aligned[["value_1", "value_2"]].max(axis=1, skipna=False) + else: + result = value_1 - value_2 + + return result.astype(float) + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries_1": {"data_type": "SERIES"}, + "timeseries_2": {"data_type": "SERIES"}, + "align_strategy": {"data_type": "STRING", "default_value": "exact"}, + "join_type": {"data_type": "STRING", "default_value": "inner"}, + "tolerance": {"data_type": "STRING", "default_value": None}, + "missing_policy": {"data_type": "STRING", "default_value": "keep_nan"}, + "merge_operation": {"data_type": "STRING", "default_value": "sum"}, + }, + "outputs": { + "combined_series": {"data_type": "SERIES"}, + }, + "name": "Align and Merge Two Time Series", + "category": "Time Series Base Components", + "description": "Align two separate time series on time and merge them into one result series.", + "version_tag": "1.0.0", + "id": "79ffe3ff-346f-4dad-ab38-ecb7b477325d", + "revision_group_id": "212ead20-b2e7-4ca7-9eb5-c38c6f32e23b", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries_1, + timeseries_2, + align_strategy="exact", + join_type="inner", + tolerance=None, + missing_policy="keep_nan", + merge_operation="sum", +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate inputs and parse the optional tolerance. + parsed_tolerance = validate_inputs( + timeseries_1, + timeseries_2, + align_strategy, + join_type, + tolerance, + missing_policy, + merge_operation, + ) + + # Step 2: Sort both series and merge duplicate timestamps by mean. + prepared_1 = prepare_series(timeseries_1) + prepared_2 = prepare_series(timeseries_2) + + # Step 3: Align both series on the selected reference index. + aligned = build_aligned_frame( + prepared_1, + prepared_2, + align_strategy, + join_type, + parsed_tolerance, + ) + + # Step 4: Apply the selected missing-value policy. + aligned = apply_missing_policy(aligned, missing_policy) + + # Step 5: Merge the aligned values into one result series. + combined = merge_aligned_values(aligned, merge_operation) + + # Step 6: Return the combined result series. + return { + "combined_series": combined, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries_1", + "filters": { + "value": '{\n "2026-03-01T10:00:00Z": 100.0,\n "2026-03-01T10:05:00Z": 105.0,\n "2026-03-01T10:10:00Z": 110.0\n}' + }, + }, + { + "workflow_input_name": "timeseries_2", + "filters": { + "value": '{\n "2026-03-01T10:00:00Z": 20.0,\n "2026-03-01T10:05:00Z": 22.0,\n "2026-03-01T10:10:00Z": 24.0\n}' + }, + }, + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries_1", + "filters": { + "value": '{\n "2026-03-01T10:00:00Z": 100.0,\n "2026-03-01T10:05:00Z": 105.0,\n "2026-03-01T10:10:00Z": 110.0\n}' + }, + }, + { + "workflow_input_name": "timeseries_2", + "filters": { + "value": '{\n "2026-03-01T10:00:00Z": 20.0,\n "2026-03-01T10:05:00Z": 22.0,\n "2026-03-01T10:10:00Z": 24.0\n}' + }, + }, + ] +} diff --git a/runtime/transformations/components/time-series-base-components/detect_jumps_in_time_series_100_8701e13f-faf4-4a7e-bb5b-724da41d4508.py b/runtime/transformations/components/time-series-base-components/detect_jumps_in_time_series_100_8701e13f-faf4-4a7e-bb5b-724da41d4508.py new file mode 100644 index 000000000..3c7a9cf42 --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/detect_jumps_in_time_series_100_8701e13f-faf4-4a7e-bb5b-724da41d4508.py @@ -0,0 +1,760 @@ +"""Documentation for Detect Jumps in Time Series + +# Detect Jumps in Time Series + +## Description +Component to detect jumps in time series data. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. Index must be datetime, values numeric. +- **method** (String, default value: "robust_zscore_on_diff"): + Jump detection method. One of "threshold_on_derivative", + "robust_zscore_on_diff", "absolute_change". + - `threshold_on_derivative`: detects jumps via strong changes between + consecutive values. This is the more specialized option. + Best suited for: + - clean signals with low noise + - fast online checks when simple step detection is enough + - data where jumps appear as clear single-change events + - `robust_zscore_on_diff`: scores changes robustly against outliers + (median/MAD), so isolated spikes are less likely to be treated as jumps. + This is the recommended default for most users. + Best suited for: + - noisy industrial sensor data + - data with occasional spikes/outliers + - cases where robust detection is preferred over maximum sensitivity + - `absolute_change`: detects jumps via a fixed minimum value change in the + original units of the time series. + Best suited for: + - signals with a known minimum relevant jump size + - engineering use cases with a clear physical threshold + - cases where a direct threshold in signal units is preferred +- **sensitivity** (String, default value: "medium"): + Controls how easily jumps are accepted after the internal auto-threshold is + calculated. This is not an absolute jump size. It only changes how strict + the automatically inferred threshold is. + - `low`: stricter detection, usually only clearer and stronger jumps + - `medium`: balanced default behavior + - `high`: more sensitive detection, so smaller or more borderline jumps + are more likely to be accepted +- **sensitivity_factor** (Float, default value: null): + Optional direct factor for the internally inferred threshold. If this input + is set, it overrides `sensitivity`. For `method="absolute_change"`, this + input is ignored. + - values larger than `1.0` make detection stricter + - values smaller than `1.0` make detection more sensitive +- **min_jump_size** (Float, default value: null): + Minimum absolute value change that creates a jump candidate. This input is + required for `method="absolute_change"` and ignored for the other methods. +- **min_distance_time** (String, default value: null): + Optional minimum time distance between two reported jumps, for example + `5min`, `30min`, or `2h`. If this input is not set, the component uses + twice the typical sampling interval of the series. +- **direction** (String, default value: "both"): + Event direction filter. One of "both", "up", "down". +- **smoothing_before** (Boolean, default value: False): + If True, applies a mild moving average before detection. This can help for + noisy signals, but it can also make the detected jump timestamp less sharp. + +## Outputs +- **jump_mask** (Pandas Series): + Boolean mask with detected jump positions. A jump is marked at the + candidate timestamp where the strong change occurs, but only if the + following points confirm a stable new level. + +## Details +1. The input series is sorted by timestamp, duplicate timestamps are merged by mean. +2. Optional smoothing is applied to stabilize detection. +3. A jump score is calculated using the selected method. +4. Transitions over unusually large time gaps are excluded from jump scoring, + so a jump is not inferred purely across a long data gap. +5. The score is compared against a robust internally determined threshold that + is scaled by `sensitivity` or by `sensitivity_factor` if it is set. For + `method="absolute_change"`, `min_jump_size` is used instead. +6. Candidates are filtered by direction and persistent post-jump behavior. +7. A candidate is kept at its original change timestamp, but only if the next + five points confirm a sufficiently stable new level. This helps suppress + short spikes that immediately return to the old level. +8. Remaining candidates are reduced by minimum time distance between events. +9. The final jump mask is returned. + +## Recommended Usage +Use the defaults first: `method="robust_zscore_on_diff"`, +`sensitivity="medium"`, and `smoothing_before=false`. + +If too many jumps are detected, use `sensitivity="low"` first. If clear jumps +are missed, use `sensitivity="high"`. If finer tuning is needed, set +`sensitivity_factor` directly. Enable `smoothing_before` only when the +signal is visibly noisy and small fluctuations create false candidates. + +If a fixed jump size in the original signal units is known, use +`method="absolute_change"` together with `min_jump_size`. + +For a real jump, the component expects a persistent level change. A single +large spike is usually rejected if the values return quickly to the previous +level. + +## Example +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 10.0, + "2026-03-01T01:00:00Z": 10.1, + "2026-03-01T02:00:00Z": 10.0, + "2026-03-01T03:00:00Z": 10.2, + "2026-03-01T04:00:00Z": 10.1, + "2026-03-01T05:00:00Z": 10.0, + "2026-03-01T06:00:00Z": 10.2, + "2026-03-01T07:00:00Z": 10.1, + "2026-03-01T08:00:00Z": 10.0, + "2026-03-01T15:00:00Z": 18.0, + "2026-03-01T16:00:00Z": 18.1, + "2026-03-01T17:00:00Z": 18.0, + "2026-03-01T18:00:00Z": 18.2, + "2026-03-01T19:00:00Z": 18.1, + "2026-03-01T20:00:00Z": 18.0, + "2026-03-01T21:00:00Z": 18.1, + "2026-03-01T22:00:00Z": 18.0, + "2026-03-01T23:00:00Z": 31.0, + "2026-03-02T00:00:00Z": 31.2, + "2026-03-02T01:00:00Z": 31.1, + "2026-03-02T02:00:00Z": 31.0, + "2026-03-02T03:00:00Z": 31.1, + "2026-03-02T04:00:00Z": 31.0, + "2026-03-02T05:00:00Z": 31.2 + }, + "method": "robust_zscore_on_diff", + "sensitivity": "medium", + "sensitivity_factor": null, + "min_jump_size": null, + "min_distance_time": null, + "direction": "both", + "smoothing_before": false +} +``` + +Expected output: +```json +{ + "jump_mask": { + "2026-03-01T00:00:00Z": false, + "2026-03-01T01:00:00Z": false, + "2026-03-01T02:00:00Z": false, + "2026-03-01T03:00:00Z": false, + "2026-03-01T04:00:00Z": false, + "2026-03-01T05:00:00Z": false, + "2026-03-01T06:00:00Z": false, + "2026-03-01T07:00:00Z": false, + "2026-03-01T08:00:00Z": false, + "2026-03-01T15:00:00Z": false, + "2026-03-01T16:00:00Z": false, + "2026-03-01T17:00:00Z": false, + "2026-03-01T18:00:00Z": false, + "2026-03-01T19:00:00Z": false, + "2026-03-01T20:00:00Z": false, + "2026-03-01T21:00:00Z": false, + "2026-03-01T22:00:00Z": false, + "2026-03-01T23:00:00Z": true, + "2026-03-02T00:00:00Z": false, + "2026-03-02T01:00:00Z": false, + "2026-03-02T02:00:00Z": false, + "2026-03-02T03:00:00Z": false, + "2026-03-02T04:00:00Z": false, + "2026-03-02T05:00:00Z": false + } +} +``` + +Second example with `method="absolute_change"`: +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 10.0, + "2026-03-01T01:00:00Z": 10.2, + "2026-03-01T02:00:00Z": 10.1, + "2026-03-01T03:00:00Z": 18.5, + "2026-03-01T04:00:00Z": 18.6, + "2026-03-01T05:00:00Z": 18.4, + "2026-03-01T06:00:00Z": 18.5 + }, + "method": "absolute_change", + "sensitivity": "medium", + "sensitivity_factor": null, + "min_jump_size": 5.0, + "min_distance_time": null, + "direction": "both", + "smoothing_before": false +} +``` + +Expected output: +```json +{ + "jump_mask": { + "2026-03-01T00:00:00Z": false, + "2026-03-01T01:00:00Z": false, + "2026-03-01T02:00:00Z": false, + "2026-03-01T03:00:00Z": true, + "2026-03-01T04:00:00Z": false, + "2026-03-01T05:00:00Z": false, + "2026-03-01T06:00:00Z": false + } +} +``` +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + +# Set fixed parameters +SMOOTHING_WINDOW = 5 +PERSISTENCE_POINTS = 5 +PERSISTENCE_TOLERANCE_FACTOR = 1.5 +PERSISTENCE_LOOKBACK_POINTS = 10 +SPIKE_REBOUND_POINTS = 1 +SPIKE_REBOUND_RATIO = 0.6 +MAX_ALLOWED_GAP_FACTOR = 3.0 +SENSITIVITY_FACTORS = { + "low": 1.25, + "medium": 1.0, + "high": 0.8, +} + + +def validate_and_normalize_inputs( + timeseries: pd.Series, + method: str, + sensitivity: str, + sensitivity_factor: float | None, + min_jump_size: float | None, + min_distance_time: str | None, + direction: str, +) -> None: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not isinstance(timeseries.index, pd.DatetimeIndex): + raise ComponentInputValidationException( + "timeseries index must be a pandas DatetimeIndex", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + valid_methods = { + "threshold_on_derivative", + "robust_zscore_on_diff", + "absolute_change", + } + if method not in valid_methods: + raise ComponentInputValidationException( + f"method must be one of {sorted(valid_methods)}", + error_code="422", + invalid_component_inputs=["method"], + ) + + if direction not in {"both", "up", "down"}: + raise ComponentInputValidationException( + "direction must be one of 'both', 'up', 'down'", + error_code="422", + invalid_component_inputs=["direction"], + ) + + if sensitivity not in SENSITIVITY_FACTORS: + raise ComponentInputValidationException( + f"sensitivity must be one of {sorted(SENSITIVITY_FACTORS)}", + error_code="422", + invalid_component_inputs=["sensitivity"], + ) + + if sensitivity_factor is not None: + if not isinstance(sensitivity_factor, int | float) or not np.isfinite( + float(sensitivity_factor) + ): + raise ComponentInputValidationException( + "sensitivity_factor must be a finite number", + error_code="422", + invalid_component_inputs=["sensitivity_factor"], + ) + if float(sensitivity_factor) <= 0: + raise ComponentInputValidationException( + "sensitivity_factor must be greater than zero", + error_code="422", + invalid_component_inputs=["sensitivity_factor"], + ) + + if method == "absolute_change": + if min_jump_size is None: + raise ComponentInputValidationException( + "min_jump_size must be set when method is 'absolute_change'", + error_code="422", + invalid_component_inputs=["min_jump_size"], + ) + if not isinstance(min_jump_size, int | float) or not np.isfinite(float(min_jump_size)): + raise ComponentInputValidationException( + "min_jump_size must be a finite number", + error_code="422", + invalid_component_inputs=["min_jump_size"], + ) + if float(min_jump_size) <= 0: + raise ComponentInputValidationException( + "min_jump_size must be greater than zero", + error_code="422", + invalid_component_inputs=["min_jump_size"], + ) + + if min_distance_time is not None: + try: + min_distance_delta = pd.to_timedelta(min_distance_time) + except ValueError as exc: + raise ComponentInputValidationException( + "min_distance_time must be a valid fixed timedelta string like '5min', '30min', or '2h'", + error_code="422", + invalid_component_inputs=["min_distance_time"], + ) from exc + if min_distance_delta <= pd.Timedelta(0): + raise ComponentInputValidationException( + "min_distance_time must be greater than zero", + error_code="422", + invalid_component_inputs=["min_distance_time"], + ) + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared + + +def calculate_dt_seconds(index: pd.Index) -> pd.Series: + diffs = index.to_series().diff().dt.total_seconds() + diffs = diffs.replace(0.0, np.nan) + return diffs + + +def infer_typical_dt_seconds(index: pd.Index) -> float | None: + dt_seconds = calculate_dt_seconds(index).dropna() + positive_dt_seconds = dt_seconds[dt_seconds > 0] + if positive_dt_seconds.empty: + return None + typical_dt_seconds = float(positive_dt_seconds.median()) + if not np.isfinite(typical_dt_seconds) or typical_dt_seconds <= 0: + return None + return typical_dt_seconds + + +def resolve_min_distance_time( + index: pd.Index, + min_distance_time: str | None, +) -> pd.Timedelta: + if min_distance_time is not None: + return pd.to_timedelta(min_distance_time) + + typical_dt_seconds = infer_typical_dt_seconds(index) + if typical_dt_seconds is None: + return pd.Timedelta(0) + return pd.to_timedelta(2.0 * typical_dt_seconds, unit="s") + + +def build_large_gap_mask( + index: pd.Index, + max_allowed_gap_factor: float, +) -> pd.Series: + large_gap_mask = pd.Series(False, index=index) + typical_dt_seconds = infer_typical_dt_seconds(index) + if typical_dt_seconds is None: + return large_gap_mask + + dt_seconds = calculate_dt_seconds(index) + gap_limit_seconds = typical_dt_seconds * max_allowed_gap_factor + return dt_seconds > gap_limit_seconds + + +def apply_smoothing( + series: pd.Series, + smoothing_before: bool, + smoothing_window: int, +) -> pd.Series: + if not smoothing_before: + return series + return series.rolling(window=smoothing_window, min_periods=1).mean() + + +def calculate_difference_per_second(series: pd.Series) -> pd.Series: + diff_signal = series.diff() + dt_seconds = calculate_dt_seconds(series.index) + per_second = diff_signal / dt_seconds + return per_second.replace([np.inf, -np.inf], np.nan) + + +def robust_auto_threshold(score: pd.Series) -> float: + valid = score.dropna() + if valid.empty: + return 0.0 + med = float(valid.median()) + mad = float((valid - med).abs().median()) + sigma = 1.4826 * mad + if sigma == 0.0 or np.isnan(sigma): + q25 = float(valid.quantile(0.25)) + q75 = float(valid.quantile(0.75)) + iqr = q75 - q25 + sigma = iqr / 1.349 if iqr > 0 else 0.0 + if sigma == 0.0 or np.isnan(sigma): + # Fallback for almost-constant signals with rare large jumps/spikes. + # We intentionally keep this low and rely on persistence filters afterwards. + return max(med, float(valid.quantile(0.90))) + return med + 3.5 * sigma + + +def apply_sensitivity_to_threshold( + threshold: float, + sensitivity: str, + sensitivity_factor: float | None, +) -> float: + factor = ( + float(sensitivity_factor) + if sensitivity_factor is not None + else SENSITIVITY_FACTORS[sensitivity] + ) + return float(threshold) * factor + + +def passes_direction(magnitude: float, direction: str) -> bool: + if direction == "both": + return True + if direction == "up": + return magnitude > 0 + return magnitude < 0 + + +def enforce_min_distance( + candidates: pd.Index, + magnitudes: pd.Series, + min_distance_time: pd.Timedelta, +) -> pd.Index: + if len(candidates) == 0: + return candidates + + kept: list[pd.Timestamp] = [] + last_kept_ts: pd.Timestamp | None = None + for ts in candidates: + if last_kept_ts is None or ts - last_kept_ts >= min_distance_time: + kept.append(ts) + last_kept_ts = ts + elif abs(float(magnitudes.loc[ts])) > abs(float(magnitudes.loc[last_kept_ts])): + kept[-1] = ts + last_kept_ts = ts + + return pd.Index(kept) + + +def remove_spike_rebound_candidates( + candidates: pd.Index, + magnitudes: pd.Series, + index_positions: pd.Series, + rebound_points: int, + rebound_ratio: float, +) -> pd.Index: + """Drop spike-like candidates that are followed or preceded by a quick opposite rebound.""" + if len(candidates) == 0: + return candidates + + kept: list[pd.Timestamp] = [] + for ts in candidates: + mag = float(magnitudes.loc[ts]) if pd.notna(magnitudes.loc[ts]) else 0.0 + if mag == 0.0: + continue + + pos = int(index_positions.loc[ts]) + lo = max(0, pos - rebound_points) + hi = min(len(magnitudes) - 1, pos + rebound_points) + neighborhood = magnitudes.iloc[lo : hi + 1].drop(labels=[ts], errors="ignore") + + has_strong_opposite_rebound = False + for rebound_mag in neighborhood.dropna().to_numpy(dtype=float): + opposite_sign = (mag > 0 and rebound_mag < 0) or (mag < 0 and rebound_mag > 0) + strong_enough = abs(rebound_mag) >= rebound_ratio * abs(mag) + if opposite_sign and strong_enough: + has_strong_opposite_rebound = True + break + + if not has_strong_opposite_rebound: + kept.append(ts) + + return pd.Index(kept) + + +def filter_persistent_jumps( + events_idx: pd.Index, + series: pd.Series, + index_positions: pd.Series, + lookback_points: int, + persistence_points: int, + tolerance_factor: float, + large_gap_mask: pd.Series, +) -> pd.Index: + if len(events_idx) == 0: + return events_idx + + gap_positions = np.flatnonzero(large_gap_mask.to_numpy(dtype=bool)) + kept: list[pd.Timestamp] = [] + for ts in events_idx: + pos = int(index_positions.loc[ts]) + previous_gap_positions = gap_positions[gap_positions <= pos] + previous_gap_pos = ( + int(previous_gap_positions[-1]) if len(previous_gap_positions) > 0 else -1 + ) + pre_start = max(previous_gap_pos + 1, pos - lookback_points) + pre_values = series.iloc[pre_start:pos].dropna() + + next_gap_positions = gap_positions[gap_positions > pos] + next_gap_pos = int(next_gap_positions[0]) if len(next_gap_positions) > 0 else len(series) + post_end = min(next_gap_pos, pos + 1 + persistence_points) + post_values = series.iloc[pos + 1 : post_end].dropna() + + if len(pre_values) == 0 or len(post_values) < persistence_points: + continue + + pre_level = float(pre_values.median()) + post_level = float(post_values.median()) + post_spread = float(post_values.std(ddof=0)) + tolerance = max(tolerance_factor * post_spread, 1e-9) + + # A persistent jump needs a meaningful level change. + if abs(post_level - pre_level) <= tolerance: + continue + + # The new level must remain stable within a tolerance band. + stable_ratio = ((post_values - post_level).abs() <= tolerance).mean() + if stable_ratio < 2 / 3: + continue + + # Reject short spikes that quickly return near the old level. + rebound_values = series.iloc[ + pos + 1 + persistence_points : min(next_gap_pos, pos + 1 + 2 * persistence_points) + ].dropna() + if len(rebound_values) >= 1: + returns_to_old = (rebound_values - pre_level).abs() <= tolerance + if returns_to_old.any(): + continue + + kept.append(ts) + + return pd.Index(kept) + + +def detect_threshold_on_derivative( + series: pd.Series, + sensitivity: str, + sensitivity_factor: float | None, +) -> tuple[pd.Series, float]: + derivative = calculate_difference_per_second(series) + large_gap_mask = build_large_gap_mask(series.index, MAX_ALLOWED_GAP_FACTOR) + derivative = derivative.mask(large_gap_mask) + score = derivative.abs() + used_threshold = apply_sensitivity_to_threshold( + robust_auto_threshold(score), sensitivity, sensitivity_factor + ) + return score, used_threshold + + +def detect_robust_zscore_on_diff( + series: pd.Series, + sensitivity: str, + sensitivity_factor: float | None, +) -> tuple[pd.Series, pd.Series, float]: + diff_signal = series.diff() + large_gap_mask = build_large_gap_mask(series.index, MAX_ALLOWED_GAP_FACTOR) + diff_signal = diff_signal.mask(large_gap_mask) + med = diff_signal.median() + mad = (diff_signal - med).abs().median() + scale = 1.4826 * mad + if scale == 0 or np.isnan(scale): + scale = diff_signal.std() + if scale == 0 or np.isnan(scale): + z = pd.Series(0.0, index=series.index) + else: + z = (diff_signal - med) / scale + score = z.abs() + used_threshold = apply_sensitivity_to_threshold( + robust_auto_threshold(score), sensitivity, sensitivity_factor + ) + return score, diff_signal, used_threshold + + +def detect_absolute_change( + series: pd.Series, + min_jump_size: float, +) -> tuple[pd.Series, pd.Series, float]: + diff_signal = series.diff() + large_gap_mask = build_large_gap_mask(series.index, MAX_ALLOWED_GAP_FACTOR) + diff_signal = diff_signal.mask(large_gap_mask) + score = diff_signal.abs() + return score, diff_signal, float(min_jump_size) + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "method": {"data_type": "STRING", "default_value": "robust_zscore_on_diff"}, + "sensitivity": {"data_type": "STRING", "default_value": "medium"}, + "sensitivity_factor": {"data_type": "FLOAT", "default_value": None}, + "min_jump_size": {"data_type": "FLOAT", "default_value": None}, + "min_distance_time": {"data_type": "STRING", "default_value": None}, + "direction": {"data_type": "STRING", "default_value": "both"}, + "smoothing_before": {"data_type": "BOOLEAN", "default_value": False}, + }, + "outputs": { + "jump_mask": {"data_type": "SERIES"}, + }, + "name": "Detect Jumps in Time Series", + "category": "Time Series Base Components", + "description": "Detect jumps and return a jump mask.", + "version_tag": "1.0.0", + "id": "8701e13f-faf4-4a7e-bb5b-724da41d4508", + "revision_group_id": "70ce7c79-ce2d-4ca2-81a1-80a0920b5bd8", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + method="robust_zscore_on_diff", + sensitivity="medium", + sensitivity_factor=None, + min_jump_size=None, + min_distance_time=None, + direction="both", + smoothing_before=False, +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate and normalize user inputs. + validate_and_normalize_inputs( + timeseries, + method, + sensitivity, + sensitivity_factor, + min_jump_size, + min_distance_time, + direction, + ) + + # Step 2: Prepare input series (sort index, merge duplicate timestamps). + prepared = prepare_series(timeseries) + + # Step 3: Optionally smooth the series before jump scoring. + smoothed = apply_smoothing(prepared, smoothing_before, SMOOTHING_WINDOW) + large_gap_mask = build_large_gap_mask(smoothed.index, MAX_ALLOWED_GAP_FACTOR) + resolved_min_distance_time = resolve_min_distance_time(smoothed.index, min_distance_time) + + # Step 4: Calculate score and magnitudes for the selected method. + if method == "threshold_on_derivative": + score, used_threshold = detect_threshold_on_derivative( + smoothed, sensitivity, sensitivity_factor + ) + magnitudes = calculate_difference_per_second(smoothed) + elif method == "absolute_change": + score, diff_signal, used_threshold = detect_absolute_change(smoothed, float(min_jump_size)) + magnitudes = diff_signal + else: + score, diff_signal, used_threshold = detect_robust_zscore_on_diff( + smoothed, sensitivity, sensitivity_factor + ) + magnitudes = diff_signal + + # Step 5: Apply threshold on score to get initial candidate jumps. + candidate_mask = score > used_threshold + candidate_mask = candidate_mask.fillna(False) + + # Step 6: Optionally filter candidates by jump direction. + if direction != "both": + direction_mask = magnitudes.apply( + lambda x: passes_direction(float(x) if pd.notna(x) else 0.0, direction) + ) + candidate_mask = candidate_mask & direction_mask + + # Step 7: Convert surviving candidates into candidate timestamps. + candidate_index = candidate_mask[candidate_mask].index + positions = pd.Series(np.arange(len(smoothed.index)), index=smoothed.index) + + # Step 8: Remove spike-like candidates with a strong opposite rebound nearby. + candidate_index = remove_spike_rebound_candidates( + candidate_index, + magnitudes, + positions, + SPIKE_REBOUND_POINTS, + SPIKE_REBOUND_RATIO, + ) + + # Step 9: Enforce persistence to suppress short spike-like events. + candidate_index = filter_persistent_jumps( + candidate_index, + smoothed, + positions, + PERSISTENCE_LOOKBACK_POINTS, + PERSISTENCE_POINTS, + PERSISTENCE_TOLERANCE_FACTOR, + large_gap_mask, + ) + + # Step 10: Enforce minimum distance and keep strongest nearby event. + filtered_index = enforce_min_distance(candidate_index, magnitudes, resolved_min_distance_time) + + # Step 11: Build output mask. + jump_mask = pd.Series(False, index=smoothed.index) + if len(filtered_index) > 0: + jump_mask.loc[filtered_index] = True + + return { + "jump_mask": jump_mask, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 10.0,\n "2026-03-01T01:00:00Z": 10.1,\n "2026-03-01T02:00:00Z": 10.0,\n "2026-03-01T03:00:00Z": 10.2,\n "2026-03-01T04:00:00Z": 10.1,\n "2026-03-01T05:00:00Z": 10.0,\n "2026-03-01T06:00:00Z": 10.2,\n "2026-03-01T07:00:00Z": 10.1,\n "2026-03-01T08:00:00Z": 10.0,\n "2026-03-01T15:00:00Z": 18.0,\n "2026-03-01T16:00:00Z": 18.1,\n "2026-03-01T17:00:00Z": 18.0,\n "2026-03-01T18:00:00Z": 18.2,\n "2026-03-01T19:00:00Z": 18.1,\n "2026-03-01T20:00:00Z": 18.0,\n "2026-03-01T21:00:00Z": 18.1,\n "2026-03-01T22:00:00Z": 18.0,\n "2026-03-01T23:00:00Z": 31.0,\n "2026-03-02T00:00:00Z": 31.2,\n "2026-03-02T01:00:00Z": 31.1,\n "2026-03-02T02:00:00Z": 31.0,\n "2026-03-02T03:00:00Z": 31.1,\n "2026-03-02T04:00:00Z": 31.0,\n "2026-03-02T05:00:00Z": 31.2\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 10.0,\n "2026-03-01T01:00:00Z": 10.1,\n "2026-03-01T02:00:00Z": 10.0,\n "2026-03-01T03:00:00Z": 10.2,\n "2026-03-01T04:00:00Z": 10.1,\n "2026-03-01T05:00:00Z": 10.0,\n "2026-03-01T06:00:00Z": 10.2,\n "2026-03-01T07:00:00Z": 10.1,\n "2026-03-01T08:00:00Z": 10.0,\n "2026-03-01T15:00:00Z": 18.0,\n "2026-03-01T16:00:00Z": 18.1,\n "2026-03-01T17:00:00Z": 18.0,\n "2026-03-01T18:00:00Z": 18.2,\n "2026-03-01T19:00:00Z": 18.1,\n "2026-03-01T20:00:00Z": 18.0,\n "2026-03-01T21:00:00Z": 18.1,\n "2026-03-01T22:00:00Z": 18.0,\n "2026-03-01T23:00:00Z": 31.0,\n "2026-03-02T00:00:00Z": 31.2,\n "2026-03-02T01:00:00Z": 31.1,\n "2026-03-02T02:00:00Z": 31.0,\n "2026-03-02T03:00:00Z": 31.1,\n "2026-03-02T04:00:00Z": 31.0,\n "2026-03-02T05:00:00Z": 31.2\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/detect_outliers_in_time_series_100_ed0943ec-a30f-4677-87f5-7b5120d86b1f.py b/runtime/transformations/components/time-series-base-components/detect_outliers_in_time_series_100_ed0943ec-a30f-4677-87f5-7b5120d86b1f.py new file mode 100644 index 000000000..ee9c42133 --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/detect_outliers_in_time_series_100_ed0943ec-a30f-4677-87f5-7b5120d86b1f.py @@ -0,0 +1,474 @@ +"""Documentation for Detect Outliers in Time Series + +# Detect Outliers in Time Series + +## Description +Component to detect outliers using a moving-window median absolute deviation +(MAD) band around the local median. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric. +- **infer_window_size** (Boolean, default value: True): + If `True`, the component infers a suitable time-based window size from the + sampling of the input series. The inference is based on the median time + difference between consecutive datapoints and the configured minimum number + of datapoints per window. If `False`, the value of `window_size` is used as + provided. +- **window_size** (String, default value: "5h"): + Size of the centered time window around each datapoint. All values inside + this window are used to estimate the local normal range. Only timedelta + strings such as `15min`, `1h`, `5h`, or `7D` are supported. This input is + only used when `infer_window_size=False`. +- **min_num_datapoints_in_window** (Integer, default value: 10): + Minimum number of datapoints required inside a window before the component + calculates a median and a MAD. If fewer points are available, no outlier + decision is made for that timestamp. +- **mad_scaling_factor** (Float, default value: 4.4478): + Multiplier for the calculated MAD. Larger values create a wider accepted + band and therefore mark fewer points as outliers. The value `1.4836` makes + the MAD comparable to a standard deviation. The default `4.4478` equals + `3 * 1.4826` and therefore corresponds roughly to a robust 3-sigma band. +- **min_band_width_factor** (Float, default value: 1.0): + Safety factor for the minimum band width. It prevents the band from + becoming unrealistically narrow in very stable phases. +- **direction** (String, default value: "both"): + Controls which type of outliers should be detected. Use `"both"` for both + sides, `"high"` for only unusually high values, or `"low"` for only + unusually low values. + +## Outputs +- **outlier_mask** (Pandas Series): + Boolean series. `True` means the datapoint is treated as an outlier. + `False` means the datapoint stays inside the accepted band. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. If `infer_window_size=True`, the component infers the window size from the + median time difference between consecutive datapoints, the configured + `min_num_datapoints_in_window`, and an internal buffer factor of `1.4`. +3. A centered moving median is calculated for each timestamp. +4. A centered moving MAD is calculated for the same windows. +5. The MAD is scaled by `mad_scaling_factor`. +6. A minimum band width is enforced using `min_band_width_factor`. +7. A datapoint is marked as an outlier if its distance to the local median is + larger than the rolling deviation. +8. The `direction` setting can restrict detection to only unusually high + values, only unusually low values, or both. +9. If too few datapoints are available in a window, no outlier decision is made + for that point and the outlier mask is set to `False`. +10. The component returns the boolean outlier mask. + +## Example +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 1.058, + "2026-03-01T00:20:48Z": 0.699, + "2026-03-01T00:41:37Z": 1.08, + "2026-03-01T01:02:26Z": 1.054, + "2026-03-01T01:23:15Z": 0.763, + "2026-03-01T01:44:04Z": 1.034, + "2026-03-01T02:04:53Z": 0.965, + "2026-03-01T02:25:42Z": 0.601, + "2026-03-01T02:46:31Z": 0.933, + "2026-03-01T03:07:20Z": 1.081, + "2026-03-01T03:28:09Z": 2.1, + "2026-03-01T03:48:58Z": 0.988, + "2026-03-01T04:09:47Z": 0.741, + "2026-03-01T04:30:36Z": 0.647, + "2026-03-01T04:51:25Z": 0.556, + "2026-03-01T05:12:14Z": 0.453, + "2026-03-01T05:33:03Z": 1.009, + "2026-03-01T05:53:52Z": 1.72, + "2026-03-01T06:14:41Z": 1.002, + "2026-03-01T06:56:19Z": 0.857, + "2026-03-01T07:17:08Z": 0.864, + "2026-03-01T07:37:57Z": 0.606, + "2026-03-01T07:58:46Z": 0.899, + "2026-03-01T08:40:24Z": 0.62, + "2026-03-01T10:03:40Z": 0.721, + "2026-03-01T10:24:29Z": 1.193, + "2026-03-01T10:45:18Z": 0.833, + "2026-03-01T11:06:07Z": 2.06, + "2026-03-01T11:26:56Z": 0.68, + "2026-03-01T11:47:45Z": 1.136, + "2026-03-01T12:08:34Z": 0.62, + "2026-03-01T12:29:23Z": 0.946, + "2026-03-01T12:50:12Z": 0.746, + "2026-03-01T13:11:01Z": 0.833, + "2026-03-01T13:31:50Z": 0.857, + "2026-03-01T14:13:28Z": 0.947, + "2026-03-01T14:34:17Z": 0.841, + "2026-03-01T14:55:06Z": 0.668, + "2026-03-01T15:15:55Z": 0.675, + "2026-03-01T15:36:44Z": 0.84, + "2026-03-01T15:57:33Z": 0.821, + "2026-03-01T16:18:22Z": 0.625, + "2026-03-01T16:39:11Z": 1.155, + "2026-03-01T17:00:00Z": 0.968 + }, + "infer_window_size": true +} +``` +""" + +from __future__ import annotations + +import math + +import numpy as np +import numpy.typing as npt +import pandas as pd +from numba import njit + +from hdutils import ComponentInputValidationException + +LARGE_SERIES_NUMBA_THRESHOLD = 10000 +INFER_WINDOW_SIZE_BUFFER_FACTOR = 1.4 + + +def parse_fixed_window_size(window_size: str) -> str: + try: + offset = pd.tseries.frequencies.to_offset(window_size) + except ValueError as exc: + raise ComponentInputValidationException( + "window_size must be a valid timedelta string like '15min', '1h', or '7D'", + error_code="422", + invalid_component_inputs=["window_size"], + ) from exc + + try: + _ = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + "window_size must be a fixed timedelta string like '15min', '1h', or '7D'", + error_code="422", + invalid_component_inputs=["window_size"], + ) from exc + + return window_size + + +def validate_inputs( + timeseries: pd.Series, + infer_window_size: bool, + window_size: str, + min_num_datapoints_in_window: int, + mad_scaling_factor: float, + min_band_width_factor: float, + direction: str, +) -> None: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not np.isfinite(timeseries.dropna().to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + "timeseries must not contain inf or -inf values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not isinstance(infer_window_size, bool): + raise ComponentInputValidationException( + "infer_window_size must be true or false", + error_code="422", + invalid_component_inputs=["infer_window_size"], + ) + if not infer_window_size: + if not isinstance(window_size, str): + raise ComponentInputValidationException( + "window_size must be a pandas time window string", + error_code="422", + invalid_component_inputs=["window_size"], + ) + parse_fixed_window_size(window_size) + if not isinstance(min_num_datapoints_in_window, int) or min_num_datapoints_in_window < 1: + raise ComponentInputValidationException( + "min_num_datapoints_in_window must be an integer >= 1", + error_code="422", + invalid_component_inputs=["min_num_datapoints_in_window"], + ) + for value, input_name in ( + (mad_scaling_factor, "mad_scaling_factor"), + (min_band_width_factor, "min_band_width_factor"), + ): + if not isinstance(value, (int, float)): + raise ComponentInputValidationException( + f"{input_name} must be a finite number", + error_code="422", + invalid_component_inputs=[input_name], + ) + if not np.isfinite(float(value)): + raise ComponentInputValidationException( + f"{input_name} must be a finite number", + error_code="422", + invalid_component_inputs=[input_name], + ) + if float(mad_scaling_factor) <= 0: + raise ComponentInputValidationException( + "mad_scaling_factor must be > 0", + error_code="422", + invalid_component_inputs=["mad_scaling_factor"], + ) + if float(min_band_width_factor) < 0: + raise ComponentInputValidationException( + "min_band_width_factor must be >= 0", + error_code="422", + invalid_component_inputs=["min_band_width_factor"], + ) + if direction not in {"both", "high", "low"}: + raise ComponentInputValidationException( + "direction must be one of 'both', 'high', 'low'", + error_code="422", + invalid_component_inputs=["direction"], + ) + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared + + +def calculate_window_size_in_seconds( + series: pd.Series, + min_num_desired_datapoints_in_window: int, + buffer_factor: float, +) -> float: + median_diff = series.sort_index().index.to_series().diff().median() + if pd.isna(median_diff): + raise ComponentInputValidationException( + "timeseries must contain at least two timestamps to infer window_size", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + median_diff_seconds = float(median_diff.total_seconds()) + if median_diff_seconds <= 0: + raise ComponentInputValidationException( + "timeseries must have a positive median time difference to infer window_size", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + return median_diff_seconds * (min_num_desired_datapoints_in_window * buffer_factor) + + +def create_pandas_frequency_string(number_of_seconds: float) -> str: + rounded_seconds = max(1, math.ceil(number_of_seconds)) + return f"{rounded_seconds}s" + + +def resolve_window_size( + series: pd.Series, + infer_window_size: bool, + window_size: str, + min_num_datapoints_in_window: int, +) -> str: + if not infer_window_size: + return parse_fixed_window_size(window_size) + + inferred_window_size = calculate_window_size_in_seconds( + series=series, + min_num_desired_datapoints_in_window=min_num_datapoints_in_window, + buffer_factor=INFER_WINDOW_SIZE_BUFFER_FACTOR, + ) + return parse_fixed_window_size(create_pandas_frequency_string(inferred_window_size)) + + +def median_absolute_deviation(data: npt.ArrayLike) -> np.float64: + return np.median(np.abs(data - np.median(data))) + + +@njit +def median_absolute_deviation_numba(data: npt.ArrayLike) -> np.float64: + return np.median(np.abs(data - np.median(data))) + + +def calculate_rolling_median_absolute_deviation( + series: pd.Series, + mad_scaling_factor: float, + window_size: str, + min_num_datapoints_in_window: int, +) -> pd.Series: + rolling = series.rolling( + window=window_size, + min_periods=min_num_datapoints_in_window, + center=True, + ) + if len(series) < LARGE_SERIES_NUMBA_THRESHOLD: + return mad_scaling_factor * rolling.apply(median_absolute_deviation, raw=True) + return mad_scaling_factor * rolling.apply( + median_absolute_deviation_numba, + raw=True, + engine="numba", + ) + + +def calculate_band_filter_statistics( + series: pd.Series, + window_size: str, + min_num_datapoints_in_window: int, + mad_scaling_factor: float, + min_band_width_factor: float, + direction: str, +) -> pd.Series: + statistics = series.to_frame(name="values") + + statistics["band_center"] = series.rolling( + window=window_size, + min_periods=min_num_datapoints_in_window, + center=True, + ).median() + + statistics["rolling_deviation"] = calculate_rolling_median_absolute_deviation( + series=series, + mad_scaling_factor=mad_scaling_factor, + window_size=window_size, + min_num_datapoints_in_window=min_num_datapoints_in_window, + ) + + min_width = np.median(statistics["rolling_deviation"].dropna()) * min_band_width_factor + statistics.loc[ + statistics["rolling_deviation"] < min_width, + "rolling_deviation", + ] = min_width + + deviation_from_center = series - statistics["band_center"] + if direction == "both": + statistics["outlier_mask"] = np.abs(deviation_from_center) > statistics["rolling_deviation"] + elif direction == "high": + statistics["outlier_mask"] = deviation_from_center > statistics["rolling_deviation"] + else: + statistics["outlier_mask"] = deviation_from_center < -statistics["rolling_deviation"] + + statistics.loc[ + statistics["band_center"].isna(), + "outlier_mask", + ] = False + + return statistics["outlier_mask"] + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "infer_window_size": {"data_type": "BOOLEAN", "default_value": True}, + "window_size": {"data_type": "STRING", "default_value": "5h"}, + "min_num_datapoints_in_window": {"data_type": "INT", "default_value": 10}, + "mad_scaling_factor": {"data_type": "FLOAT", "default_value": 4.4478}, + "min_band_width_factor": {"data_type": "FLOAT", "default_value": 1.0}, + "direction": {"data_type": "STRING", "default_value": "both"}, + }, + "outputs": { + "outlier_mask": {"data_type": "SERIES"}, + }, + "name": "Detect Outliers in Time Series", + "category": "Time Series Base Components", + "description": "Detect outliers using a moving-window MAD band filter.", + "version_tag": "1.0.0", + "id": "ed0943ec-a30f-4677-87f5-7b5120d86b1f", + "revision_group_id": "bf8daf45-6d28-4c4a-bba8-290b3c5bacc5", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + infer_window_size=True, + window_size="5h", + min_num_datapoints_in_window=10, + mad_scaling_factor=4.4478, + min_band_width_factor=1.0, + direction="both", +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate the user inputs. + validate_inputs( + timeseries, + infer_window_size, + window_size, + min_num_datapoints_in_window, + mad_scaling_factor, + min_band_width_factor, + direction, + ) + + # Step 2: Sort the input series and merge duplicate timestamps. + prepared = prepare_series(timeseries) + + # Step 3: Resolve the effective window size, either inferred or user-defined. + effective_window_size = resolve_window_size( + series=prepared, + infer_window_size=infer_window_size, + window_size=window_size, + min_num_datapoints_in_window=min_num_datapoints_in_window, + ) + + # Step 4: Calculate the moving window band statistics and outlier mask. + outlier_mask = calculate_band_filter_statistics( + series=prepared, + window_size=effective_window_size, + min_num_datapoints_in_window=min_num_datapoints_in_window, + mad_scaling_factor=mad_scaling_factor, + min_band_width_factor=min_band_width_factor, + direction=direction, + ) + + # Step 5: Return the outlier mask. + return { + "outlier_mask": outlier_mask, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 1.058,\n "2026-03-01T00:20:48Z": 0.699,\n "2026-03-01T00:41:37Z": 1.08,\n "2026-03-01T01:02:26Z": 1.054,\n "2026-03-01T01:23:15Z": 0.763,\n "2026-03-01T01:44:04Z": 1.034,\n "2026-03-01T02:04:53Z": 0.965,\n "2026-03-01T02:25:42Z": 0.601,\n "2026-03-01T02:46:31Z": 0.933,\n "2026-03-01T03:07:20Z": 1.081,\n "2026-03-01T03:28:09Z": 2.1,\n "2026-03-01T03:48:58Z": 0.988,\n "2026-03-01T04:09:47Z": 0.741,\n "2026-03-01T04:30:36Z": 0.647,\n "2026-03-01T04:51:25Z": 0.556,\n "2026-03-01T05:12:14Z": 0.453,\n "2026-03-01T05:33:03Z": 1.009,\n "2026-03-01T05:53:52Z": 1.72,\n "2026-03-01T06:14:41Z": 1.002,\n "2026-03-01T06:56:19Z": 0.857,\n "2026-03-01T07:17:08Z": 0.864,\n "2026-03-01T07:37:57Z": 0.606,\n "2026-03-01T07:58:46Z": 0.899,\n "2026-03-01T08:40:24Z": 0.62,\n "2026-03-01T10:03:40Z": 0.721,\n "2026-03-01T10:24:29Z": 1.193,\n "2026-03-01T10:45:18Z": 0.833,\n "2026-03-01T11:06:07Z": 2.06,\n "2026-03-01T11:26:56Z": 0.68,\n "2026-03-01T11:47:45Z": 1.136,\n "2026-03-01T12:08:34Z": 0.62,\n "2026-03-01T12:29:23Z": 0.946,\n "2026-03-01T12:50:12Z": 0.746,\n "2026-03-01T13:11:01Z": 0.833,\n "2026-03-01T13:31:50Z": 0.857,\n "2026-03-01T14:13:28Z": 0.947,\n "2026-03-01T14:34:17Z": 0.841,\n "2026-03-01T14:55:06Z": 0.668,\n "2026-03-01T15:15:55Z": 0.675,\n "2026-03-01T15:36:44Z": 0.84,\n "2026-03-01T15:57:33Z": 0.821,\n "2026-03-01T16:18:22Z": 0.625,\n "2026-03-01T16:39:11Z": 1.155,\n "2026-03-01T17:00:00Z": 0.968\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 1.058,\n "2026-03-01T00:20:48Z": 0.699,\n "2026-03-01T00:41:37Z": 1.08,\n "2026-03-01T01:02:26Z": 1.054,\n "2026-03-01T01:23:15Z": 0.763,\n "2026-03-01T01:44:04Z": 1.034,\n "2026-03-01T02:04:53Z": 0.965,\n "2026-03-01T02:25:42Z": 0.601,\n "2026-03-01T02:46:31Z": 0.933,\n "2026-03-01T03:07:20Z": 1.081,\n "2026-03-01T03:28:09Z": 2.1,\n "2026-03-01T03:48:58Z": 0.988,\n "2026-03-01T04:09:47Z": 0.741,\n "2026-03-01T04:30:36Z": 0.647,\n "2026-03-01T04:51:25Z": 0.556,\n "2026-03-01T05:12:14Z": 0.453,\n "2026-03-01T05:33:03Z": 1.009,\n "2026-03-01T05:53:52Z": 1.72,\n "2026-03-01T06:14:41Z": 1.002,\n "2026-03-01T06:56:19Z": 0.857,\n "2026-03-01T07:17:08Z": 0.864,\n "2026-03-01T07:37:57Z": 0.606,\n "2026-03-01T07:58:46Z": 0.899,\n "2026-03-01T08:40:24Z": 0.62,\n "2026-03-01T10:03:40Z": 0.721,\n "2026-03-01T10:24:29Z": 1.193,\n "2026-03-01T10:45:18Z": 0.833,\n "2026-03-01T11:06:07Z": 2.06,\n "2026-03-01T11:26:56Z": 0.68,\n "2026-03-01T11:47:45Z": 1.136,\n "2026-03-01T12:08:34Z": 0.62,\n "2026-03-01T12:29:23Z": 0.946,\n "2026-03-01T12:50:12Z": 0.746,\n "2026-03-01T13:11:01Z": 0.833,\n "2026-03-01T13:31:50Z": 0.857,\n "2026-03-01T14:13:28Z": 0.947,\n "2026-03-01T14:34:17Z": 0.841,\n "2026-03-01T14:55:06Z": 0.668,\n "2026-03-01T15:15:55Z": 0.675,\n "2026-03-01T15:36:44Z": 0.84,\n "2026-03-01T15:57:33Z": 0.821,\n "2026-03-01T16:18:22Z": 0.625,\n "2026-03-01T16:39:11Z": 1.155,\n "2026-03-01T17:00:00Z": 0.968\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/detect_threshold_crossings_in_time_series_100_076036e9-9298-4deb-a0c4-4d2180149d9f.py b/runtime/transformations/components/time-series-base-components/detect_threshold_crossings_in_time_series_100_076036e9-9298-4deb-a0c4-4d2180149d9f.py new file mode 100644 index 000000000..ebaa745cd --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/detect_threshold_crossings_in_time_series_100_076036e9-9298-4deb-a0c4-4d2180149d9f.py @@ -0,0 +1,309 @@ +"""Documentation for Detect Threshold Crossings in Time Series + +# Detect Threshold Crossings in Time Series + +## Description +Component to detect when a time series crosses a fixed threshold. It can detect +upward crossings, downward crossings, or both directions. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric. +- **threshold** (Float): + The threshold value to test against. +- **crossing_type** (String, default value: "both"): + Selects which crossing direction is detected. + - `upward`: detect crossings from below the threshold to above it. + - `downward`: detect crossings from above the threshold to below it. + - `both`: detect both directions. +- **inclusiveness** (String, default value: "strict"): + Controls whether touching the threshold already counts as a crossing. + - `strict`: the series must move from one side of the threshold to the other. + - `inclusive`: reaching the threshold already counts as a crossing. + +## Outputs +- **crossing_mask** (Pandas Series): + Bool series with `True` at detected crossing timestamps and `False` + otherwise. A crossing is marked on the right timestamp of the interval in + which the crossing becomes visible. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. The component checks each pair of consecutive timestamps. +3. If one of the two endpoint values is missing, no crossing is emitted for + that interval. +4. Depending on `crossing_type`, upward crossings, downward crossings, or both + are detected. +5. With `strict`, the series must move from one side of the threshold to the other. +6. With `inclusive`, reaching the threshold already counts as a crossing. +7. The crossing result of an interval is written to the right timestamp of that + interval. + +## Example +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 8.0, + "2026-03-01T01:00:00Z": 9.5, + "2026-03-01T02:00:00Z": 10.5, + "2026-03-01T03:00:00Z": 11.0, + "2026-03-01T04:00:00Z": 9.0 + }, + "threshold": 10.0, + "crossing_type": "both", + "inclusiveness": "strict" +} +``` + +Expected output: +```json +{ + "crossing_mask": { + "2026-03-01T01:00:00Z": false, + "2026-03-01T02:00:00Z": true, + "2026-03-01T03:00:00Z": false, + "2026-03-01T04:00:00Z": true + } +} +``` + +Second example with `inclusive` upward detection: +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 8.0, + "2026-03-01T01:00:00Z": 10.0, + "2026-03-01T02:00:00Z": 10.0, + "2026-03-01T03:00:00Z": 11.0 + }, + "threshold": 10.0, + "crossing_type": "upward", + "inclusiveness": "inclusive" +} +``` + +Expected output: +```json +{ + "crossing_mask": { + "2026-03-01T01:00:00Z": true, + "2026-03-01T02:00:00Z": false, + "2026-03-01T03:00:00Z": false + } +} +``` +""" + +from __future__ import annotations + +import math + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + +CROSSING_TYPES = {"upward", "downward", "both"} +INCLUSIVENESS_MODES = {"strict", "inclusive"} + + +def validate_inputs( + timeseries: pd.Series, + threshold: float, + crossing_type: str, + inclusiveness: str, +) -> float: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not np.isfinite(timeseries.dropna().to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + "timeseries must not contain inf or -inf values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.dropna().shape[0] < 2: + raise ComponentInputValidationException( + "timeseries must contain at least two non-missing numeric values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + try: + parsed_threshold = float(threshold) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "threshold must be a numeric value", + error_code="422", + invalid_component_inputs=["threshold"], + ) from exc + + if not math.isfinite(parsed_threshold): + raise ComponentInputValidationException( + "threshold must be finite", + error_code="422", + invalid_component_inputs=["threshold"], + ) + + if crossing_type not in CROSSING_TYPES: + raise ComponentInputValidationException( + f"crossing_type must be one of {sorted(CROSSING_TYPES)}", + error_code="422", + invalid_component_inputs=["crossing_type"], + ) + + if inclusiveness not in INCLUSIVENESS_MODES: + raise ComponentInputValidationException( + f"inclusiveness must be one of {sorted(INCLUSIVENESS_MODES)}", + error_code="422", + invalid_component_inputs=["inclusiveness"], + ) + + return parsed_threshold + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared.astype(float) + + +def is_upward_crossing( + previous_value: float, current_value: float, threshold: float, inclusiveness: str +) -> bool: + if inclusiveness == "strict": + return previous_value < threshold and current_value > threshold + return previous_value < threshold and current_value >= threshold + + +def is_downward_crossing( + previous_value: float, current_value: float, threshold: float, inclusiveness: str +) -> bool: + if inclusiveness == "strict": + return previous_value > threshold and current_value < threshold + return previous_value > threshold and current_value <= threshold + + +def detect_crossings( + series: pd.Series, + threshold: float, + crossing_type: str, + inclusiveness: str, +) -> pd.Series: + results: dict[pd.Timestamp, bool] = {} + + for idx in range(1, len(series)): + current_time = series.index[idx] + previous_value = series.iloc[idx - 1] + current_value = series.iloc[idx] + + if pd.isna(previous_value) or pd.isna(current_value): + results[current_time] = False + continue + + upward = is_upward_crossing( + float(previous_value), float(current_value), threshold, inclusiveness + ) + downward = is_downward_crossing( + float(previous_value), float(current_value), threshold, inclusiveness + ) + + if crossing_type == "upward": + results[current_time] = upward + elif crossing_type == "downward": + results[current_time] = downward + else: + results[current_time] = upward or downward + + return pd.Series(results, dtype=bool) + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "threshold": {"data_type": "FLOAT"}, + "crossing_type": {"data_type": "STRING", "default_value": "both"}, + "inclusiveness": {"data_type": "STRING", "default_value": "strict"}, + }, + "outputs": { + "crossing_mask": {"data_type": "SERIES"}, + }, + "name": "Detect Threshold Crossings in Time Series", + "category": "Time Series Base Components", + "description": "Detect upward and downward threshold crossings on a time series.", + "version_tag": "1.0.0", + "id": "076036e9-9298-4deb-a0c4-4d2180149d9f", + "revision_group_id": "cd3d8d41-05ae-4bb4-9266-6350981d08fd", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main(*, timeseries, threshold, crossing_type="both", inclusiveness="strict"): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate inputs and parse the threshold. + threshold = validate_inputs(timeseries, threshold, crossing_type, inclusiveness) + + # Step 2: Sort the series and merge duplicate timestamps by mean. + prepared = prepare_series(timeseries) + + # Step 3: Detect threshold crossings on consecutive intervals. + crossing_mask = detect_crossings(prepared, threshold, crossing_type, inclusiveness) + + # Step 4: Return the crossing mask. + return { + "crossing_mask": crossing_mask, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 8.2,\n "2026-03-01T01:00:00Z": 8.9,\n "2026-03-01T02:00:00Z": 9.7,\n "2026-03-01T03:00:00Z": 10.4,\n "2026-03-01T04:00:00Z": 11.1,\n "2026-03-01T05:00:00Z": 10.6,\n "2026-03-01T06:00:00Z": 9.8,\n "2026-03-01T07:00:00Z": 9.1,\n "2026-03-01T08:00:00Z": 8.7,\n "2026-03-01T09:00:00Z": 9.4,\n "2026-03-01T10:00:00Z": 9.8,\n "2026-03-01T11:00:00Z": 10.2,\n "2026-03-01T12:00:00Z": 10.9,\n "2026-03-01T13:00:00Z": 10.1,\n "2026-03-01T14:00:00Z": 9.6,\n "2026-03-01T15:00:00Z": 9.0,\n "2026-03-01T16:00:00Z": 9.4,\n "2026-03-01T17:00:00Z": 9.7,\n "2026-03-01T18:00:00Z": 9.9,\n "2026-03-01T19:00:00Z": 9.2,\n "2026-03-01T20:00:00Z": 8.8\n}' + }, + }, + {"workflow_input_name": "threshold", "filters": {"value": "10.0"}}, + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 8.2,\n "2026-03-01T01:00:00Z": 8.9,\n "2026-03-01T02:00:00Z": 9.7,\n "2026-03-01T03:00:00Z": 10.4,\n "2026-03-01T04:00:00Z": 11.1,\n "2026-03-01T05:00:00Z": 10.6,\n "2026-03-01T06:00:00Z": 9.8,\n "2026-03-01T07:00:00Z": 9.1,\n "2026-03-01T08:00:00Z": 8.7,\n "2026-03-01T09:00:00Z": 9.4,\n "2026-03-01T10:00:00Z": 9.8,\n "2026-03-01T11:00:00Z": 10.2,\n "2026-03-01T12:00:00Z": 10.9,\n "2026-03-01T13:00:00Z": 10.1,\n "2026-03-01T14:00:00Z": 9.6,\n "2026-03-01T15:00:00Z": 9.0,\n "2026-03-01T16:00:00Z": 9.4,\n "2026-03-01T17:00:00Z": 9.7,\n "2026-03-01T18:00:00Z": 9.9,\n "2026-03-01T19:00:00Z": 9.2,\n "2026-03-01T20:00:00Z": 8.8\n}' + }, + }, + {"workflow_input_name": "threshold", "filters": {"value": "10.0"}}, + ] +} diff --git a/runtime/transformations/components/time-series-base-components/differentiate_time_series_100_2da73403-dfc7-4168-bbf4-324c4e6c97db.py b/runtime/transformations/components/time-series-base-components/differentiate_time_series_100_2da73403-dfc7-4168-bbf4-324c4e6c97db.py new file mode 100644 index 000000000..f4ba24274 --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/differentiate_time_series_100_2da73403-dfc7-4168-bbf4-324c4e6c97db.py @@ -0,0 +1,391 @@ +"""Documentation for Differentiate Time Series + +# Differentiate Time Series + +## Description +Component to calculate either the difference or the rate of change between +neighboring timestamps of a time series on a datetime axis. It supports +irregular sampling, defensive gap handling, and an optional maximum allowed +time gap. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric. +- **output_mode** (String, default value: "rate"): + Selects which result is returned. + - `rate`: returns the change per selected `time_unit`. + - `difference`: returns the plain value difference between two neighboring + used timestamps. +- **time_unit** (String, default value: "h"): + Time unit used to scale the rate result. Must be one of `s`, `min`, `h`, + or `d`. + For example, if the values are in `kWh` and `time_unit="h"`, the rate + output is in `kW`. For `output_mode="difference"`, this input has no + effect. +- **gap_handling** (String, default value: "break"): + Controls how missing values are handled during differentiation. + - `break`: only use directly neighboring timestamps. If one of the two + endpoint values is missing, no derivative result is created for that + interval. This is the safer default. + - `ignore`: skip missing observations and connect the remaining valid + points. This is a more aggressive option because it differentiates across + gaps. +- **max_gap** (String, default value: null): + Optional maximum time gap allowed between two used timestamps. If the time + difference is larger than this value, no result is created for that + interval. Use a fixed timedelta string such as `2h`, `30min`, or `1D`, or + `null` to disable this check. + +## Outputs +- **result_series** (Pandas Series): + The differentiated result series. Each result is placed on the right + timestamp of the interval it was calculated from. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. Depending on `gap_handling`, missing values are either treated as hard + breaks or skipped when selecting point pairs. +3. If `max_gap` is set, intervals larger than this threshold are skipped even + if both endpoint values are valid. +4. For `output_mode="difference"`, the component calculates `y_i - y_(i-1)`. +5. For `output_mode="rate"`, the component calculates `(y_i - y_(i-1)) / dt` + and scales the result to the selected `time_unit`. + For `output_mode="difference"`, `time_unit` is ignored. +6. The result of an interval is placed on the right timestamp of that interval. +7. If the series should first be put on a regular time grid, use + `resample_time_series` upstream. If missing values should first be handled + explicitly, use `handle_gaps_and_missing_data` upstream and keep + `gap_handling="break"` here. + +## Example +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 10.0, + "2026-03-01T01:00:00Z": 11.0, + "2026-03-01T02:00:00Z": 11.5, + "2026-03-01T03:00:00Z": null, + "2026-03-01T04:00:00Z": 15.5, + "2026-03-01T05:00:00Z": 16.0 + }, + "output_mode": "rate", + "time_unit": "h", + "gap_handling": "break", + "max_gap": "2h" +} +``` + +Expected output: +```json +{ + "result_series": { + "2026-03-01T01:00:00Z": 1.0, + "2026-03-01T02:00:00Z": 0.5, + "2026-03-01T05:00:00Z": 0.5 + } +} +``` + +Second example with `difference` and `ignore`: +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 100.0, + "2026-03-01T01:00:00Z": 101.0, + "2026-03-01T02:00:00Z": null, + "2026-03-01T03:00:00Z": 106.0, + "2026-03-01T04:00:00Z": 108.0 + }, + "output_mode": "difference", + "time_unit": "h", + "gap_handling": "ignore", + "max_gap": "3h" +} +``` + +Expected output: +```json +{ + "result_series": { + "2026-03-01T01:00:00Z": 1.0, + "2026-03-01T03:00:00Z": 5.0, + "2026-03-01T04:00:00Z": 2.0 + } +} +``` + +Notes: +- `gap_handling="break"` is the recommended default when missing values should + stop the differentiation. +- `gap_handling="ignore"` should only be used intentionally, because it + connects valid datapoints across gaps. +- `max_gap` is useful if large time gaps should not produce a difference or + rate, even when both endpoint values are valid. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + +TIME_UNIT_IN_SECONDS = { + "s": 1.0, + "min": 60.0, + "h": 3600.0, + "d": 86400.0, +} + + +def parse_fixed_timedelta_string(value: str, input_name: str) -> pd.Timedelta: + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a valid fixed timedelta string like '30min', '2h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + try: + nanos = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '30min', '2h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + return pd.to_timedelta(nanos, unit="ns") + + +def validate_inputs( + timeseries: pd.Series, + output_mode: str, + time_unit: str, + gap_handling: str, + max_gap: str | None, +) -> pd.Timedelta | None: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not np.isfinite(timeseries.dropna().to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + "timeseries must not contain inf or -inf values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.dropna().shape[0] < 2: + raise ComponentInputValidationException( + "timeseries must contain at least two non-missing numeric values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if output_mode not in {"rate", "difference"}: + raise ComponentInputValidationException( + "output_mode must be one of 'rate' or 'difference'", + error_code="422", + invalid_component_inputs=["output_mode"], + ) + if time_unit not in TIME_UNIT_IN_SECONDS: + raise ComponentInputValidationException( + "time_unit must be one of 's', 'min', 'h', or 'd'", + error_code="422", + invalid_component_inputs=["time_unit"], + ) + if gap_handling not in {"break", "ignore"}: + raise ComponentInputValidationException( + "gap_handling must be one of 'break' or 'ignore'", + error_code="422", + invalid_component_inputs=["gap_handling"], + ) + + parsed_max_gap = None + if max_gap is not None: + parsed_max_gap = parse_fixed_timedelta_string(max_gap, "max_gap") + if parsed_max_gap <= pd.Timedelta(0): + raise ComponentInputValidationException( + "max_gap must be greater than zero", + error_code="422", + invalid_component_inputs=["max_gap"], + ) + + return parsed_max_gap + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared.astype(float) + + +def build_effective_series(series: pd.Series, gap_handling: str) -> pd.Series: + if gap_handling == "ignore": + return series.dropna() + return series + + +def should_skip_interval( + left_time: pd.Timestamp, + right_time: pd.Timestamp, + max_gap: pd.Timedelta | None, +) -> bool: + if right_time <= left_time: + return True + if max_gap is None: + return False + return (right_time - left_time) > max_gap + + +def calculate_result_series( + series: pd.Series, + output_mode: str, + time_unit: str, + gap_handling: str, + max_gap: pd.Timedelta | None, +) -> pd.Series: + effective = build_effective_series(series, gap_handling) + results: dict[pd.Timestamp, float] = {} + scaling_seconds = TIME_UNIT_IN_SECONDS[time_unit] + + if gap_handling == "break": + for idx in range(1, len(effective)): + left_time = effective.index[idx - 1] + right_time = effective.index[idx] + left_value = effective.iloc[idx - 1] + right_value = effective.iloc[idx] + + if pd.isna(left_value) or pd.isna(right_value): + continue + if should_skip_interval(left_time, right_time, max_gap): + continue + + value_difference = float(right_value - left_value) + if output_mode == "difference": + results[right_time] = value_difference + else: + dt_seconds = (right_time - left_time).total_seconds() + results[right_time] = value_difference / (dt_seconds / scaling_seconds) + else: + for idx in range(1, len(effective)): + left_time = effective.index[idx - 1] + right_time = effective.index[idx] + left_value = float(effective.iloc[idx - 1]) + right_value = float(effective.iloc[idx]) + + if should_skip_interval(left_time, right_time, max_gap): + continue + + value_difference = right_value - left_value + if output_mode == "difference": + results[right_time] = value_difference + else: + dt_seconds = (right_time - left_time).total_seconds() + results[right_time] = value_difference / (dt_seconds / scaling_seconds) + + if not results: + return pd.Series(dtype=float, index=pd.DatetimeIndex([], dtype="datetime64[ns]")) + return pd.Series(results, dtype=float) + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "output_mode": {"data_type": "STRING", "default_value": "rate"}, + "time_unit": {"data_type": "STRING", "default_value": "h"}, + "gap_handling": {"data_type": "STRING", "default_value": "break"}, + "max_gap": {"data_type": "STRING", "default_value": None}, + }, + "outputs": { + "result_series": {"data_type": "SERIES"}, + }, + "name": "Differentiate Time Series", + "category": "Time Series Base Components", + "description": "Calculate differences or rates of change on a datetime axis.", + "version_tag": "1.0.0", + "id": "2da73403-dfc7-4168-bbf4-324c4e6c97db", + "revision_group_id": "0bc7b6d3-f607-4c7d-8c96-85188d09bdb1", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main(*, timeseries, output_mode="rate", time_unit="h", gap_handling="break", max_gap=None): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate inputs and parse the optional maximum gap. + parsed_max_gap = validate_inputs( + timeseries, + output_mode, + time_unit, + gap_handling, + max_gap, + ) + + # Step 2: Sort the series and merge duplicate timestamps by mean. + prepared = prepare_series(timeseries) + + # Step 3: Calculate the requested difference or rate series. + result = calculate_result_series( + prepared, + output_mode, + time_unit, + gap_handling, + parsed_max_gap, + ) + + # Step 4: Return the differentiated result series. + return { + "result_series": result, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 10.0,\n "2026-03-01T01:00:00Z": 11.0,\n "2026-03-01T02:00:00Z": 11.5,\n "2026-03-01T03:00:00Z": null,\n "2026-03-01T04:00:00Z": 15.5,\n "2026-03-01T05:00:00Z": 16.0\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 10.0,\n "2026-03-01T01:00:00Z": 11.0,\n "2026-03-01T02:00:00Z": 11.5,\n "2026-03-01T03:00:00Z": null,\n "2026-03-01T04:00:00Z": 15.5,\n "2026-03-01T05:00:00Z": 16.0\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/frequency_filter_time_series_100_e1d983fa-6feb-409e-a872-f9e82930864a.py b/runtime/transformations/components/time-series-base-components/frequency_filter_time_series_100_e1d983fa-6feb-409e-a872-f9e82930864a.py new file mode 100644 index 000000000..03e921805 --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/frequency_filter_time_series_100_e1d983fa-6feb-409e-a872-f9e82930864a.py @@ -0,0 +1,623 @@ +"""Documentation for Frequency Filter Time Series + +# Frequency Filter Time Series + +## Description +Component to apply a Butterworth lowpass or highpass filter to a uniformly +sampled time series. + +Use this component when the time series is already on a regular time grid and +you want to either smooth away fast fluctuations or remove slow baseline +movement. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric and finite. Missing values are not allowed because digital + filters cannot process gaps directly. +- **filter_type** (String, default value: "lowpass"): + Selects the filter behavior. Must be one of `lowpass` or `highpass`. + - `lowpass`: keeps slower changes and reduces fast fluctuations. This is + useful for smoothing. + - `highpass`: keeps faster changes and reduces slow baseline movement. + This is useful when short-term variation is more relevant than the slow + trend. +- **cutoff_period** (String, default value: "4s"): + The approximate time scale at which the filter separates slow and fast + behavior. Use a fixed timedelta string such as `10s`, `5min`, `1h`, or + `1D`. + For example, with `cutoff_period="10min"`, a lowpass filter mainly keeps + behavior slower than roughly 10 minutes, while a highpass filter mainly + keeps behavior faster than roughly 10 minutes. +- **order** (Integer, default value: 1): + Butterworth filter order. Must be a positive integer. Higher values create + a sharper separation between kept and reduced signal parts, but may also + make the result harder to interpret. The default `1` is deliberately mild. +- **forward_backward** (Boolean, default value: True): + If `True`, the filter is applied forward and backward. This avoids a time + shift in the result and is the recommended default. If `False`, a single + forward pass is used, which can shift the signal slightly in time. + +## Outputs +- **filtered_timeseries** (Pandas Series): + The filtered time series with the original timestamps. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. The remaining timestamps must have exactly constant spacing. +3. Values must be numeric and finite. Use `handle_gaps_and_missing_data` + upstream if the series contains missing values. +4. `cutoff_period` is converted internally to a cutoff frequency. +5. The cutoff must be compatible with the sampling interval. In practice, + `cutoff_period` must be larger than two sampling intervals. For example, + for 1-minute data, use a cutoff period larger than `2min`. +6. The component applies a Butterworth `lowpass` or `highpass` filter and + returns only the filtered series. + +## Recommended Workflow +For irregular or incomplete data, use this order: +1. `resample_time_series` +2. `handle_gaps_and_missing_data` +3. `frequency_filter_time_series` + +This keeps the responsibility clear: first create the time grid, then handle +missing values, then filter the clean series. + +## Example +```json +{ + "timeseries": { + "2025-01-01 00:00:00.000Z": 0.0, + "2025-01-01 00:00:00.100Z": 0.776, + "2025-01-01 00:00:00.200Z": 1.327, + "2025-01-01 00:00:00.300Z": 1.513, + "2025-01-01 00:00:00.400Z": 1.334, + "2025-01-01 00:00:00.500Z": 0.927, + "2025-01-01 00:00:00.600Z": 0.517, + "2025-01-01 00:00:00.700Z": 0.326, + "2025-01-01 00:00:00.800Z": 0.494, + "2025-01-01 00:00:00.900Z": 1.02, + "2025-01-01 00:00:01.000Z": 1.763, + "2025-01-01 00:00:01.100Z": 2.5, + "2025-01-01 00:00:01.200Z": 3.005, + "2025-01-01 00:00:01.300Z": 3.138, + "2025-01-01 00:00:01.400Z": 2.899, + "2025-01-01 00:00:01.500Z": 2.427, + "2025-01-01 00:00:01.600Z": 1.945, + "2025-01-01 00:00:01.700Z": 1.678, + "2025-01-01 00:00:01.800Z": 1.763, + "2025-01-01 00:00:01.900Z": 2.202, + "2025-01-01 00:00:02.000Z": 2.853, + "2025-01-01 00:00:02.100Z": 3.494, + "2025-01-01 00:00:02.200Z": 3.898, + "2025-01-01 00:00:02.300Z": 3.927, + "2025-01-01 00:00:02.400Z": 3.582, + "2025-01-01 00:00:02.500Z": 3.0, + "2025-01-01 00:00:02.600Z": 2.406, + "2025-01-01 00:00:02.700Z": 2.025, + "2025-01-01 00:00:02.800Z": 1.996, + "2025-01-01 00:00:02.900Z": 2.318, + "2025-01-01 00:00:03.000Z": 2.853, + "2025-01-01 00:00:03.100Z": 3.377, + "2025-01-01 00:00:03.200Z": 3.666, + "2025-01-01 00:00:03.300Z": 3.58, + "2025-01-01 00:00:03.400Z": 3.121, + "2025-01-01 00:00:03.500Z": 2.427, + "2025-01-01 00:00:03.600Z": 1.724, + "2025-01-01 00:00:03.700Z": 1.236, + "2025-01-01 00:00:03.800Z": 1.103, + "2025-01-01 00:00:03.900Z": 1.324, + "2025-01-01 00:00:04.000Z": 1.763, + "2025-01-01 00:00:04.100Z": 2.195, + "2025-01-01 00:00:04.200Z": 2.396, + "2025-01-01 00:00:04.300Z": 2.228, + "2025-01-01 00:00:04.400Z": 1.692, + "2025-01-01 00:00:04.500Z": 0.927, + "2025-01-01 00:00:04.600Z": 0.158, + "2025-01-01 00:00:04.700Z": -0.389, + "2025-01-01 00:00:04.800Z": -0.575, + "2025-01-01 00:00:04.900Z": -0.399, + "2025-01-01 00:00:05.000Z": -0.0, + "2025-01-01 00:00:05.100Z": 0.399, + "2025-01-01 00:00:05.200Z": 0.575, + "2025-01-01 00:00:05.300Z": 0.389, + "2025-01-01 00:00:05.400Z": -0.158, + "2025-01-01 00:00:05.500Z": -0.927, + "2025-01-01 00:00:05.600Z": -1.692, + "2025-01-01 00:00:05.700Z": -2.228, + "2025-01-01 00:00:05.800Z": -2.396, + "2025-01-01 00:00:05.900Z": -2.195, + "2025-01-01 00:00:06.000Z": -1.763, + "2025-01-01 00:00:06.100Z": -1.324, + "2025-01-01 00:00:06.200Z": -1.103, + "2025-01-01 00:00:06.300Z": -1.236, + "2025-01-01 00:00:06.400Z": -1.724, + "2025-01-01 00:00:06.500Z": -2.427, + "2025-01-01 00:00:06.600Z": -3.121, + "2025-01-01 00:00:06.700Z": -3.58, + "2025-01-01 00:00:06.800Z": -3.666, + "2025-01-01 00:00:06.900Z": -3.377, + "2025-01-01 00:00:07.000Z": -2.853, + "2025-01-01 00:00:07.100Z": -2.318, + "2025-01-01 00:00:07.200Z": -1.996, + "2025-01-01 00:00:07.300Z": -2.025, + "2025-01-01 00:00:07.400Z": -2.406, + "2025-01-01 00:00:07.500Z": -3.0, + "2025-01-01 00:00:07.600Z": -3.582, + "2025-01-01 00:00:07.700Z": -3.927, + "2025-01-01 00:00:07.800Z": -3.898, + "2025-01-01 00:00:07.900Z": -3.494, + "2025-01-01 00:00:08.000Z": -2.853, + "2025-01-01 00:00:08.100Z": -2.202, + "2025-01-01 00:00:08.200Z": -1.763, + "2025-01-01 00:00:08.300Z": -1.678, + "2025-01-01 00:00:08.400Z": -1.945, + "2025-01-01 00:00:08.500Z": -2.427, + "2025-01-01 00:00:08.600Z": -2.899, + "2025-01-01 00:00:08.700Z": -3.138, + "2025-01-01 00:00:08.800Z": -3.005, + "2025-01-01 00:00:08.900Z": -2.5, + "2025-01-01 00:00:09.000Z": -1.763, + "2025-01-01 00:00:09.100Z": -1.02, + "2025-01-01 00:00:09.200Z": -0.494, + "2025-01-01 00:00:09.300Z": -0.326, + "2025-01-01 00:00:09.400Z": -0.517, + "2025-01-01 00:00:09.500Z": -0.927, + "2025-01-01 00:00:09.600Z": -1.334, + "2025-01-01 00:00:09.700Z": -1.513, + "2025-01-01 00:00:09.800Z": -1.327, + "2025-01-01 00:00:09.900Z": -0.776, + "2025-01-01 00:00:10.000Z": -0.0, + "2025-01-01 00:00:10.100Z": 0.776, + "2025-01-01 00:00:10.200Z": 1.327, + "2025-01-01 00:00:10.300Z": 1.513, + "2025-01-01 00:00:10.400Z": 1.334, + "2025-01-01 00:00:10.500Z": 0.927, + "2025-01-01 00:00:10.600Z": 0.517, + "2025-01-01 00:00:10.700Z": 0.326, + "2025-01-01 00:00:10.800Z": 0.494, + "2025-01-01 00:00:10.900Z": 1.02, + "2025-01-01 00:00:11.000Z": 1.763, + "2025-01-01 00:00:11.100Z": 2.5, + "2025-01-01 00:00:11.200Z": 3.005, + "2025-01-01 00:00:11.300Z": 3.138, + "2025-01-01 00:00:11.400Z": 2.899, + "2025-01-01 00:00:11.500Z": 2.427, + "2025-01-01 00:00:11.600Z": 1.945, + "2025-01-01 00:00:11.700Z": 1.678, + "2025-01-01 00:00:11.800Z": 1.763, + "2025-01-01 00:00:11.900Z": 2.202, + "2025-01-01 00:00:12.000Z": 2.853, + "2025-01-01 00:00:12.100Z": 3.494, + "2025-01-01 00:00:12.200Z": 3.898, + "2025-01-01 00:00:12.300Z": 3.927, + "2025-01-01 00:00:12.400Z": 3.582, + "2025-01-01 00:00:12.500Z": 3.0, + "2025-01-01 00:00:12.600Z": 2.406, + "2025-01-01 00:00:12.700Z": 2.025, + "2025-01-01 00:00:12.800Z": 1.996, + "2025-01-01 00:00:12.900Z": 2.318, + "2025-01-01 00:00:13.000Z": 2.853, + "2025-01-01 00:00:13.100Z": 3.377, + "2025-01-01 00:00:13.200Z": 3.666, + "2025-01-01 00:00:13.300Z": 3.58, + "2025-01-01 00:00:13.400Z": 3.121, + "2025-01-01 00:00:13.500Z": 2.427, + "2025-01-01 00:00:13.600Z": 1.724, + "2025-01-01 00:00:13.700Z": 1.236, + "2025-01-01 00:00:13.800Z": 1.103, + "2025-01-01 00:00:13.900Z": 1.324, + "2025-01-01 00:00:14.000Z": 1.763, + "2025-01-01 00:00:14.100Z": 2.195, + "2025-01-01 00:00:14.200Z": 2.396, + "2025-01-01 00:00:14.300Z": 2.228, + "2025-01-01 00:00:14.400Z": 1.692, + "2025-01-01 00:00:14.500Z": 0.927, + "2025-01-01 00:00:14.600Z": 0.158, + "2025-01-01 00:00:14.700Z": -0.389, + "2025-01-01 00:00:14.800Z": -0.575, + "2025-01-01 00:00:14.900Z": -0.399, + "2025-01-01 00:00:15.000Z": -0.0, + "2025-01-01 00:00:15.100Z": 0.399, + "2025-01-01 00:00:15.200Z": 0.575, + "2025-01-01 00:00:15.300Z": 0.389, + "2025-01-01 00:00:15.400Z": -0.158, + "2025-01-01 00:00:15.500Z": -0.927, + "2025-01-01 00:00:15.600Z": -1.692, + "2025-01-01 00:00:15.700Z": -2.228, + "2025-01-01 00:00:15.800Z": -2.396, + "2025-01-01 00:00:15.900Z": -2.195, + "2025-01-01 00:00:16.000Z": -1.763, + "2025-01-01 00:00:16.100Z": -1.324, + "2025-01-01 00:00:16.200Z": -1.103, + "2025-01-01 00:00:16.300Z": -1.236, + "2025-01-01 00:00:16.400Z": -1.724, + "2025-01-01 00:00:16.500Z": -2.427, + "2025-01-01 00:00:16.600Z": -3.121, + "2025-01-01 00:00:16.700Z": -3.58, + "2025-01-01 00:00:16.800Z": -3.666, + "2025-01-01 00:00:16.900Z": -3.377, + "2025-01-01 00:00:17.000Z": -2.853, + "2025-01-01 00:00:17.100Z": -2.318, + "2025-01-01 00:00:17.200Z": -1.996, + "2025-01-01 00:00:17.300Z": -2.025, + "2025-01-01 00:00:17.400Z": -2.406, + "2025-01-01 00:00:17.500Z": -3.0, + "2025-01-01 00:00:17.600Z": -3.582, + "2025-01-01 00:00:17.700Z": -3.927, + "2025-01-01 00:00:17.800Z": -3.898, + "2025-01-01 00:00:17.900Z": -3.494, + "2025-01-01 00:00:18.000Z": -2.853, + "2025-01-01 00:00:18.100Z": -2.202, + "2025-01-01 00:00:18.200Z": -1.763, + "2025-01-01 00:00:18.300Z": -1.678, + "2025-01-01 00:00:18.400Z": -1.945, + "2025-01-01 00:00:18.500Z": -2.427, + "2025-01-01 00:00:18.600Z": -2.899, + "2025-01-01 00:00:18.700Z": -3.138, + "2025-01-01 00:00:18.800Z": -3.005, + "2025-01-01 00:00:18.900Z": -2.5, + "2025-01-01 00:00:19.000Z": -1.763, + "2025-01-01 00:00:19.100Z": -1.02, + "2025-01-01 00:00:19.200Z": -0.494, + "2025-01-01 00:00:19.300Z": -0.326, + "2025-01-01 00:00:19.400Z": -0.517, + "2025-01-01 00:00:19.500Z": -0.927, + "2025-01-01 00:00:19.600Z": -1.334, + "2025-01-01 00:00:19.700Z": -1.513, + "2025-01-01 00:00:19.800Z": -1.327, + "2025-01-01 00:00:19.900Z": -0.776, + "2025-01-01 00:00:20.000Z": -0.0, + "2025-01-01 00:00:20.100Z": 0.776, + "2025-01-01 00:00:20.200Z": 1.327, + "2025-01-01 00:00:20.300Z": 1.513, + "2025-01-01 00:00:20.400Z": 1.334, + "2025-01-01 00:00:20.500Z": 0.927, + "2025-01-01 00:00:20.600Z": 0.517, + "2025-01-01 00:00:20.700Z": 0.326, + "2025-01-01 00:00:20.800Z": 0.494, + "2025-01-01 00:00:20.900Z": 1.02, + "2025-01-01 00:00:21.000Z": 1.763, + "2025-01-01 00:00:21.100Z": 2.5, + "2025-01-01 00:00:21.200Z": 3.005, + "2025-01-01 00:00:21.300Z": 3.138, + "2025-01-01 00:00:21.400Z": 2.899, + "2025-01-01 00:00:21.500Z": 2.427, + "2025-01-01 00:00:21.600Z": 1.945, + "2025-01-01 00:00:21.700Z": 1.678, + "2025-01-01 00:00:21.800Z": 1.763, + "2025-01-01 00:00:21.900Z": 2.202, + "2025-01-01 00:00:22.000Z": 2.853, + "2025-01-01 00:00:22.100Z": 3.494, + "2025-01-01 00:00:22.200Z": 3.898, + "2025-01-01 00:00:22.300Z": 3.927, + "2025-01-01 00:00:22.400Z": 3.582, + "2025-01-01 00:00:22.500Z": 3.0, + "2025-01-01 00:00:22.600Z": 2.406, + "2025-01-01 00:00:22.700Z": 2.025, + "2025-01-01 00:00:22.800Z": 1.996, + "2025-01-01 00:00:22.900Z": 2.318, + "2025-01-01 00:00:23.000Z": 2.853, + "2025-01-01 00:00:23.100Z": 3.377, + "2025-01-01 00:00:23.200Z": 3.666, + "2025-01-01 00:00:23.300Z": 3.58, + "2025-01-01 00:00:23.400Z": 3.121, + "2025-01-01 00:00:23.500Z": 2.427, + "2025-01-01 00:00:23.600Z": 1.724, + "2025-01-01 00:00:23.700Z": 1.236, + "2025-01-01 00:00:23.800Z": 1.103, + "2025-01-01 00:00:23.900Z": 1.324, + "2025-01-01 00:00:24.000Z": 1.763, + "2025-01-01 00:00:24.100Z": 2.195, + "2025-01-01 00:00:24.200Z": 2.396, + "2025-01-01 00:00:24.300Z": 2.228, + "2025-01-01 00:00:24.400Z": 1.692, + "2025-01-01 00:00:24.500Z": 0.927, + "2025-01-01 00:00:24.600Z": 0.158, + "2025-01-01 00:00:24.700Z": -0.389, + "2025-01-01 00:00:24.800Z": -0.575, + "2025-01-01 00:00:24.900Z": -0.399, + "2025-01-01 00:00:25.000Z": 0.0, + "2025-01-01 00:00:25.100Z": 0.399, + "2025-01-01 00:00:25.200Z": 0.575, + "2025-01-01 00:00:25.300Z": 0.389, + "2025-01-01 00:00:25.400Z": -0.158, + "2025-01-01 00:00:25.500Z": -0.927, + "2025-01-01 00:00:25.600Z": -1.692, + "2025-01-01 00:00:25.700Z": -2.228, + "2025-01-01 00:00:25.800Z": -2.396, + "2025-01-01 00:00:25.900Z": -2.195, + "2025-01-01 00:00:26.000Z": -1.763, + "2025-01-01 00:00:26.100Z": -1.324, + "2025-01-01 00:00:26.200Z": -1.103, + "2025-01-01 00:00:26.300Z": -1.236, + "2025-01-01 00:00:26.400Z": -1.724, + "2025-01-01 00:00:26.500Z": -2.427, + "2025-01-01 00:00:26.600Z": -3.121, + "2025-01-01 00:00:26.700Z": -3.58, + "2025-01-01 00:00:26.800Z": -3.666, + "2025-01-01 00:00:26.900Z": -3.377, + "2025-01-01 00:00:27.000Z": -2.853, + "2025-01-01 00:00:27.100Z": -2.318, + "2025-01-01 00:00:27.200Z": -1.996, + "2025-01-01 00:00:27.300Z": -2.025, + "2025-01-01 00:00:27.400Z": -2.406, + "2025-01-01 00:00:27.500Z": -3.0, + "2025-01-01 00:00:27.600Z": -3.582, + "2025-01-01 00:00:27.700Z": -3.927, + "2025-01-01 00:00:27.800Z": -3.898, + "2025-01-01 00:00:27.900Z": -3.494, + "2025-01-01 00:00:28.000Z": -2.853, + "2025-01-01 00:00:28.100Z": -2.202, + "2025-01-01 00:00:28.200Z": -1.763, + "2025-01-01 00:00:28.300Z": -1.678, + "2025-01-01 00:00:28.400Z": -1.945, + "2025-01-01 00:00:28.500Z": -2.427, + "2025-01-01 00:00:28.600Z": -2.899, + "2025-01-01 00:00:28.700Z": -3.138, + "2025-01-01 00:00:28.800Z": -3.005, + "2025-01-01 00:00:28.900Z": -2.5, + "2025-01-01 00:00:29.000Z": -1.763, + "2025-01-01 00:00:29.100Z": -1.02, + "2025-01-01 00:00:29.200Z": -0.494, + "2025-01-01 00:00:29.300Z": -0.326, + "2025-01-01 00:00:29.400Z": -0.517, + "2025-01-01 00:00:29.500Z": -0.927, + "2025-01-01 00:00:29.600Z": -1.334, + "2025-01-01 00:00:29.700Z": -1.513, + "2025-01-01 00:00:29.800Z": -1.327, + "2025-01-01 00:00:29.900Z": -0.776, + "2025-01-01 00:00:30.000Z": -0.0 +}, + "filter_type": "lowpass", + "cutoff_period": "4s", + "order": 1, + "forward_backward": true +} +``` + +The exact numeric values depend on the filter parameters. The important point +is that the output has the same timestamps as the input and contains the +filtered signal. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from scipy.signal import butter, filtfilt, lfilter + +from hdutils import ComponentInputValidationException + +FILTER_TYPES = {"lowpass": "low", "highpass": "high"} + + +def parse_fixed_timedelta_string(value: str, input_name: str) -> pd.Timedelta: + if not isinstance(value, str) or not value.strip(): + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '10s', '5min', or '1h'", + error_code="422", + invalid_component_inputs=[input_name], + ) + + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a valid fixed timedelta string like '10s', '5min', or '1h'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + try: + nanos = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '10s', '5min', or '1h'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + if nanos <= 0: + raise ComponentInputValidationException( + f"{input_name} must be greater than zero", + error_code="422", + invalid_component_inputs=[input_name], + ) + + return pd.to_timedelta(nanos, unit="ns") + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared + + +def validate_inputs( + timeseries: pd.Series, + filter_type: str, + cutoff_period: str, + order: int, + forward_backward: bool, +) -> pd.Timedelta: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty or timeseries.size < 2: + raise ComponentInputValidationException( + "timeseries must contain at least two samples", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if filter_type not in FILTER_TYPES: + raise ComponentInputValidationException( + "filter_type must be one of 'lowpass' or 'highpass'", + error_code="422", + invalid_component_inputs=["filter_type"], + ) + if not isinstance(order, int) or order < 1: + raise ComponentInputValidationException( + "order must be a positive integer", + error_code="422", + invalid_component_inputs=["order"], + ) + if not isinstance(forward_backward, bool): + raise ComponentInputValidationException( + "forward_backward must be a boolean", + error_code="422", + invalid_component_inputs=["forward_backward"], + ) + + return parse_fixed_timedelta_string(cutoff_period, "cutoff_period") + + +def validate_prepared_series( + prepared: pd.Series, cutoff_period: pd.Timedelta, order: int, forward_backward: bool +) -> float: + if not np.isfinite(prepared.to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + "timeseries must not contain NaN, inf, or -inf values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + idx = prepared.index + diffs_ns = (idx[1:] - idx[:-1]).astype("timedelta64[ns]").astype(np.int64) + if (diffs_ns <= 0).any(): + raise ComponentInputValidationException( + "timeseries index must be strictly increasing after duplicate timestamps are merged", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not np.all(diffs_ns == diffs_ns[0]): + raise ComponentInputValidationException( + "timeseries must have exactly constant spacing. Use resample_time_series upstream if needed", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + dt_seconds = float(diffs_ns[0]) / 1e9 + cutoff_seconds = cutoff_period.total_seconds() + if cutoff_seconds <= 2 * dt_seconds: + raise ComponentInputValidationException( + "cutoff_period must be larger than two sampling intervals", + error_code="422", + invalid_component_inputs=["cutoff_period"], + ) + + if forward_backward: + pad_length = 3 * (order + 1) + if prepared.size <= pad_length: + raise ComponentInputValidationException( + f"timeseries must contain more than {pad_length} samples when forward_backward is True and order is {order}", + error_code="422", + invalid_component_inputs=["timeseries", "order", "forward_backward"], + ) + + nyquist_frequency = 0.5 / dt_seconds + cutoff_frequency = 1.0 / cutoff_seconds + normal_frequency = cutoff_frequency / nyquist_frequency + if not (0 < normal_frequency < 1): + raise ComponentInputValidationException( + "cutoff_period is not compatible with the sampling interval", + error_code="422", + invalid_component_inputs=["cutoff_period"], + ) + + return normal_frequency + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "filter_type": {"data_type": "STRING", "default_value": "lowpass"}, + "cutoff_period": {"data_type": "STRING", "default_value": "4s"}, + "order": {"data_type": "INT", "default_value": 1}, + "forward_backward": {"data_type": "BOOLEAN", "default_value": True}, + }, + "outputs": { + "filtered_timeseries": {"data_type": "SERIES"}, + }, + "name": "Frequency Filter Time Series", + "category": "Time Series Base Components", + "description": "Apply a lowpass or highpass frequency filter to a uniformly sampled time series.", # noqa: E501 + "version_tag": "1.0.0", + "id": "e1d983fa-6feb-409e-a872-f9e82930864a", + "revision_group_id": "d92b617d-25b4-478b-87a4-47c0325c5793", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + filter_type="lowpass", + cutoff_period="4s", + order=1, + forward_backward=True, +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + cutoff_period = validate_inputs( + timeseries, + filter_type, + cutoff_period, + order, + forward_backward, + ) + + prepared = prepare_series(timeseries) + normal_frequency = validate_prepared_series(prepared, cutoff_period, order, forward_backward) + + b, a = butter(order, normal_frequency, btype=FILTER_TYPES[filter_type], analog=False) + values = prepared.to_numpy(dtype=float) + filtered_values = filtfilt(b, a, values) if forward_backward else lfilter(b, a, values) + + return { + "filtered_timeseries": pd.Series(filtered_values, index=prepared.index), + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2025-01-01 00:00:00.000Z": 0.0,\n "2025-01-01 00:00:00.100Z": 0.776,\n "2025-01-01 00:00:00.200Z": 1.327,\n "2025-01-01 00:00:00.300Z": 1.513,\n "2025-01-01 00:00:00.400Z": 1.334,\n "2025-01-01 00:00:00.500Z": 0.927,\n "2025-01-01 00:00:00.600Z": 0.517,\n "2025-01-01 00:00:00.700Z": 0.326,\n "2025-01-01 00:00:00.800Z": 0.494,\n "2025-01-01 00:00:00.900Z": 1.02,\n "2025-01-01 00:00:01.000Z": 1.763,\n "2025-01-01 00:00:01.100Z": 2.5,\n "2025-01-01 00:00:01.200Z": 3.005,\n "2025-01-01 00:00:01.300Z": 3.138,\n "2025-01-01 00:00:01.400Z": 2.899,\n "2025-01-01 00:00:01.500Z": 2.427,\n "2025-01-01 00:00:01.600Z": 1.945,\n "2025-01-01 00:00:01.700Z": 1.678,\n "2025-01-01 00:00:01.800Z": 1.763,\n "2025-01-01 00:00:01.900Z": 2.202,\n "2025-01-01 00:00:02.000Z": 2.853,\n "2025-01-01 00:00:02.100Z": 3.494,\n "2025-01-01 00:00:02.200Z": 3.898,\n "2025-01-01 00:00:02.300Z": 3.927,\n "2025-01-01 00:00:02.400Z": 3.582,\n "2025-01-01 00:00:02.500Z": 3.0,\n "2025-01-01 00:00:02.600Z": 2.406,\n "2025-01-01 00:00:02.700Z": 2.025,\n "2025-01-01 00:00:02.800Z": 1.996,\n "2025-01-01 00:00:02.900Z": 2.318,\n "2025-01-01 00:00:03.000Z": 2.853,\n "2025-01-01 00:00:03.100Z": 3.377,\n "2025-01-01 00:00:03.200Z": 3.666,\n "2025-01-01 00:00:03.300Z": 3.58,\n "2025-01-01 00:00:03.400Z": 3.121,\n "2025-01-01 00:00:03.500Z": 2.427,\n "2025-01-01 00:00:03.600Z": 1.724,\n "2025-01-01 00:00:03.700Z": 1.236,\n "2025-01-01 00:00:03.800Z": 1.103,\n "2025-01-01 00:00:03.900Z": 1.324,\n "2025-01-01 00:00:04.000Z": 1.763,\n "2025-01-01 00:00:04.100Z": 2.195,\n "2025-01-01 00:00:04.200Z": 2.396,\n "2025-01-01 00:00:04.300Z": 2.228,\n "2025-01-01 00:00:04.400Z": 1.692,\n "2025-01-01 00:00:04.500Z": 0.927,\n "2025-01-01 00:00:04.600Z": 0.158,\n "2025-01-01 00:00:04.700Z": -0.389,\n "2025-01-01 00:00:04.800Z": -0.575,\n "2025-01-01 00:00:04.900Z": -0.399,\n "2025-01-01 00:00:05.000Z": -0.0,\n "2025-01-01 00:00:05.100Z": 0.399,\n "2025-01-01 00:00:05.200Z": 0.575,\n "2025-01-01 00:00:05.300Z": 0.389,\n "2025-01-01 00:00:05.400Z": -0.158,\n "2025-01-01 00:00:05.500Z": -0.927,\n "2025-01-01 00:00:05.600Z": -1.692,\n "2025-01-01 00:00:05.700Z": -2.228,\n "2025-01-01 00:00:05.800Z": -2.396,\n "2025-01-01 00:00:05.900Z": -2.195,\n "2025-01-01 00:00:06.000Z": -1.763,\n "2025-01-01 00:00:06.100Z": -1.324,\n "2025-01-01 00:00:06.200Z": -1.103,\n "2025-01-01 00:00:06.300Z": -1.236,\n "2025-01-01 00:00:06.400Z": -1.724,\n "2025-01-01 00:00:06.500Z": -2.427,\n "2025-01-01 00:00:06.600Z": -3.121,\n "2025-01-01 00:00:06.700Z": -3.58,\n "2025-01-01 00:00:06.800Z": -3.666,\n "2025-01-01 00:00:06.900Z": -3.377,\n "2025-01-01 00:00:07.000Z": -2.853,\n "2025-01-01 00:00:07.100Z": -2.318,\n "2025-01-01 00:00:07.200Z": -1.996,\n "2025-01-01 00:00:07.300Z": -2.025,\n "2025-01-01 00:00:07.400Z": -2.406,\n "2025-01-01 00:00:07.500Z": -3.0,\n "2025-01-01 00:00:07.600Z": -3.582,\n "2025-01-01 00:00:07.700Z": -3.927,\n "2025-01-01 00:00:07.800Z": -3.898,\n "2025-01-01 00:00:07.900Z": -3.494,\n "2025-01-01 00:00:08.000Z": -2.853,\n "2025-01-01 00:00:08.100Z": -2.202,\n "2025-01-01 00:00:08.200Z": -1.763,\n "2025-01-01 00:00:08.300Z": -1.678,\n "2025-01-01 00:00:08.400Z": -1.945,\n "2025-01-01 00:00:08.500Z": -2.427,\n "2025-01-01 00:00:08.600Z": -2.899,\n "2025-01-01 00:00:08.700Z": -3.138,\n "2025-01-01 00:00:08.800Z": -3.005,\n "2025-01-01 00:00:08.900Z": -2.5,\n "2025-01-01 00:00:09.000Z": -1.763,\n "2025-01-01 00:00:09.100Z": -1.02,\n "2025-01-01 00:00:09.200Z": -0.494,\n "2025-01-01 00:00:09.300Z": -0.326,\n "2025-01-01 00:00:09.400Z": -0.517,\n "2025-01-01 00:00:09.500Z": -0.927,\n "2025-01-01 00:00:09.600Z": -1.334,\n "2025-01-01 00:00:09.700Z": -1.513,\n "2025-01-01 00:00:09.800Z": -1.327,\n "2025-01-01 00:00:09.900Z": -0.776,\n "2025-01-01 00:00:10.000Z": -0.0,\n "2025-01-01 00:00:10.100Z": 0.776,\n "2025-01-01 00:00:10.200Z": 1.327,\n "2025-01-01 00:00:10.300Z": 1.513,\n "2025-01-01 00:00:10.400Z": 1.334,\n "2025-01-01 00:00:10.500Z": 0.927,\n "2025-01-01 00:00:10.600Z": 0.517,\n "2025-01-01 00:00:10.700Z": 0.326,\n "2025-01-01 00:00:10.800Z": 0.494,\n "2025-01-01 00:00:10.900Z": 1.02,\n "2025-01-01 00:00:11.000Z": 1.763,\n "2025-01-01 00:00:11.100Z": 2.5,\n "2025-01-01 00:00:11.200Z": 3.005,\n "2025-01-01 00:00:11.300Z": 3.138,\n "2025-01-01 00:00:11.400Z": 2.899,\n "2025-01-01 00:00:11.500Z": 2.427,\n "2025-01-01 00:00:11.600Z": 1.945,\n "2025-01-01 00:00:11.700Z": 1.678,\n "2025-01-01 00:00:11.800Z": 1.763,\n "2025-01-01 00:00:11.900Z": 2.202,\n "2025-01-01 00:00:12.000Z": 2.853,\n "2025-01-01 00:00:12.100Z": 3.494,\n "2025-01-01 00:00:12.200Z": 3.898,\n "2025-01-01 00:00:12.300Z": 3.927,\n "2025-01-01 00:00:12.400Z": 3.582,\n "2025-01-01 00:00:12.500Z": 3.0,\n "2025-01-01 00:00:12.600Z": 2.406,\n "2025-01-01 00:00:12.700Z": 2.025,\n "2025-01-01 00:00:12.800Z": 1.996,\n "2025-01-01 00:00:12.900Z": 2.318,\n "2025-01-01 00:00:13.000Z": 2.853,\n "2025-01-01 00:00:13.100Z": 3.377,\n "2025-01-01 00:00:13.200Z": 3.666,\n "2025-01-01 00:00:13.300Z": 3.58,\n "2025-01-01 00:00:13.400Z": 3.121,\n "2025-01-01 00:00:13.500Z": 2.427,\n "2025-01-01 00:00:13.600Z": 1.724,\n "2025-01-01 00:00:13.700Z": 1.236,\n "2025-01-01 00:00:13.800Z": 1.103,\n "2025-01-01 00:00:13.900Z": 1.324,\n "2025-01-01 00:00:14.000Z": 1.763,\n "2025-01-01 00:00:14.100Z": 2.195,\n "2025-01-01 00:00:14.200Z": 2.396,\n "2025-01-01 00:00:14.300Z": 2.228,\n "2025-01-01 00:00:14.400Z": 1.692,\n "2025-01-01 00:00:14.500Z": 0.927,\n "2025-01-01 00:00:14.600Z": 0.158,\n "2025-01-01 00:00:14.700Z": -0.389,\n "2025-01-01 00:00:14.800Z": -0.575,\n "2025-01-01 00:00:14.900Z": -0.399,\n "2025-01-01 00:00:15.000Z": -0.0,\n "2025-01-01 00:00:15.100Z": 0.399,\n "2025-01-01 00:00:15.200Z": 0.575,\n "2025-01-01 00:00:15.300Z": 0.389,\n "2025-01-01 00:00:15.400Z": -0.158,\n "2025-01-01 00:00:15.500Z": -0.927,\n "2025-01-01 00:00:15.600Z": -1.692,\n "2025-01-01 00:00:15.700Z": -2.228,\n "2025-01-01 00:00:15.800Z": -2.396,\n "2025-01-01 00:00:15.900Z": -2.195,\n "2025-01-01 00:00:16.000Z": -1.763,\n "2025-01-01 00:00:16.100Z": -1.324,\n "2025-01-01 00:00:16.200Z": -1.103,\n "2025-01-01 00:00:16.300Z": -1.236,\n "2025-01-01 00:00:16.400Z": -1.724,\n "2025-01-01 00:00:16.500Z": -2.427,\n "2025-01-01 00:00:16.600Z": -3.121,\n "2025-01-01 00:00:16.700Z": -3.58,\n "2025-01-01 00:00:16.800Z": -3.666,\n "2025-01-01 00:00:16.900Z": -3.377,\n "2025-01-01 00:00:17.000Z": -2.853,\n "2025-01-01 00:00:17.100Z": -2.318,\n "2025-01-01 00:00:17.200Z": -1.996,\n "2025-01-01 00:00:17.300Z": -2.025,\n "2025-01-01 00:00:17.400Z": -2.406,\n "2025-01-01 00:00:17.500Z": -3.0,\n "2025-01-01 00:00:17.600Z": -3.582,\n "2025-01-01 00:00:17.700Z": -3.927,\n "2025-01-01 00:00:17.800Z": -3.898,\n "2025-01-01 00:00:17.900Z": -3.494,\n "2025-01-01 00:00:18.000Z": -2.853,\n "2025-01-01 00:00:18.100Z": -2.202,\n "2025-01-01 00:00:18.200Z": -1.763,\n "2025-01-01 00:00:18.300Z": -1.678,\n "2025-01-01 00:00:18.400Z": -1.945,\n "2025-01-01 00:00:18.500Z": -2.427,\n "2025-01-01 00:00:18.600Z": -2.899,\n "2025-01-01 00:00:18.700Z": -3.138,\n "2025-01-01 00:00:18.800Z": -3.005,\n "2025-01-01 00:00:18.900Z": -2.5,\n "2025-01-01 00:00:19.000Z": -1.763,\n "2025-01-01 00:00:19.100Z": -1.02,\n "2025-01-01 00:00:19.200Z": -0.494,\n "2025-01-01 00:00:19.300Z": -0.326,\n "2025-01-01 00:00:19.400Z": -0.517,\n "2025-01-01 00:00:19.500Z": -0.927,\n "2025-01-01 00:00:19.600Z": -1.334,\n "2025-01-01 00:00:19.700Z": -1.513,\n "2025-01-01 00:00:19.800Z": -1.327,\n "2025-01-01 00:00:19.900Z": -0.776,\n "2025-01-01 00:00:20.000Z": -0.0,\n "2025-01-01 00:00:20.100Z": 0.776,\n "2025-01-01 00:00:20.200Z": 1.327,\n "2025-01-01 00:00:20.300Z": 1.513,\n "2025-01-01 00:00:20.400Z": 1.334,\n "2025-01-01 00:00:20.500Z": 0.927,\n "2025-01-01 00:00:20.600Z": 0.517,\n "2025-01-01 00:00:20.700Z": 0.326,\n "2025-01-01 00:00:20.800Z": 0.494,\n "2025-01-01 00:00:20.900Z": 1.02,\n "2025-01-01 00:00:21.000Z": 1.763,\n "2025-01-01 00:00:21.100Z": 2.5,\n "2025-01-01 00:00:21.200Z": 3.005,\n "2025-01-01 00:00:21.300Z": 3.138,\n "2025-01-01 00:00:21.400Z": 2.899,\n "2025-01-01 00:00:21.500Z": 2.427,\n "2025-01-01 00:00:21.600Z": 1.945,\n "2025-01-01 00:00:21.700Z": 1.678,\n "2025-01-01 00:00:21.800Z": 1.763,\n "2025-01-01 00:00:21.900Z": 2.202,\n "2025-01-01 00:00:22.000Z": 2.853,\n "2025-01-01 00:00:22.100Z": 3.494,\n "2025-01-01 00:00:22.200Z": 3.898,\n "2025-01-01 00:00:22.300Z": 3.927,\n "2025-01-01 00:00:22.400Z": 3.582,\n "2025-01-01 00:00:22.500Z": 3.0,\n "2025-01-01 00:00:22.600Z": 2.406,\n "2025-01-01 00:00:22.700Z": 2.025,\n "2025-01-01 00:00:22.800Z": 1.996,\n "2025-01-01 00:00:22.900Z": 2.318,\n "2025-01-01 00:00:23.000Z": 2.853,\n "2025-01-01 00:00:23.100Z": 3.377,\n "2025-01-01 00:00:23.200Z": 3.666,\n "2025-01-01 00:00:23.300Z": 3.58,\n "2025-01-01 00:00:23.400Z": 3.121,\n "2025-01-01 00:00:23.500Z": 2.427,\n "2025-01-01 00:00:23.600Z": 1.724,\n "2025-01-01 00:00:23.700Z": 1.236,\n "2025-01-01 00:00:23.800Z": 1.103,\n "2025-01-01 00:00:23.900Z": 1.324,\n "2025-01-01 00:00:24.000Z": 1.763,\n "2025-01-01 00:00:24.100Z": 2.195,\n "2025-01-01 00:00:24.200Z": 2.396,\n "2025-01-01 00:00:24.300Z": 2.228,\n "2025-01-01 00:00:24.400Z": 1.692,\n "2025-01-01 00:00:24.500Z": 0.927,\n "2025-01-01 00:00:24.600Z": 0.158,\n "2025-01-01 00:00:24.700Z": -0.389,\n "2025-01-01 00:00:24.800Z": -0.575,\n "2025-01-01 00:00:24.900Z": -0.399,\n "2025-01-01 00:00:25.000Z": 0.0,\n "2025-01-01 00:00:25.100Z": 0.399,\n "2025-01-01 00:00:25.200Z": 0.575,\n "2025-01-01 00:00:25.300Z": 0.389,\n "2025-01-01 00:00:25.400Z": -0.158,\n "2025-01-01 00:00:25.500Z": -0.927,\n "2025-01-01 00:00:25.600Z": -1.692,\n "2025-01-01 00:00:25.700Z": -2.228,\n "2025-01-01 00:00:25.800Z": -2.396,\n "2025-01-01 00:00:25.900Z": -2.195,\n "2025-01-01 00:00:26.000Z": -1.763,\n "2025-01-01 00:00:26.100Z": -1.324,\n "2025-01-01 00:00:26.200Z": -1.103,\n "2025-01-01 00:00:26.300Z": -1.236,\n "2025-01-01 00:00:26.400Z": -1.724,\n "2025-01-01 00:00:26.500Z": -2.427,\n "2025-01-01 00:00:26.600Z": -3.121,\n "2025-01-01 00:00:26.700Z": -3.58,\n "2025-01-01 00:00:26.800Z": -3.666,\n "2025-01-01 00:00:26.900Z": -3.377,\n "2025-01-01 00:00:27.000Z": -2.853,\n "2025-01-01 00:00:27.100Z": -2.318,\n "2025-01-01 00:00:27.200Z": -1.996,\n "2025-01-01 00:00:27.300Z": -2.025,\n "2025-01-01 00:00:27.400Z": -2.406,\n "2025-01-01 00:00:27.500Z": -3.0,\n "2025-01-01 00:00:27.600Z": -3.582,\n "2025-01-01 00:00:27.700Z": -3.927,\n "2025-01-01 00:00:27.800Z": -3.898,\n "2025-01-01 00:00:27.900Z": -3.494,\n "2025-01-01 00:00:28.000Z": -2.853,\n "2025-01-01 00:00:28.100Z": -2.202,\n "2025-01-01 00:00:28.200Z": -1.763,\n "2025-01-01 00:00:28.300Z": -1.678,\n "2025-01-01 00:00:28.400Z": -1.945,\n "2025-01-01 00:00:28.500Z": -2.427,\n "2025-01-01 00:00:28.600Z": -2.899,\n "2025-01-01 00:00:28.700Z": -3.138,\n "2025-01-01 00:00:28.800Z": -3.005,\n "2025-01-01 00:00:28.900Z": -2.5,\n "2025-01-01 00:00:29.000Z": -1.763,\n "2025-01-01 00:00:29.100Z": -1.02,\n "2025-01-01 00:00:29.200Z": -0.494,\n "2025-01-01 00:00:29.300Z": -0.326,\n "2025-01-01 00:00:29.400Z": -0.517,\n "2025-01-01 00:00:29.500Z": -0.927,\n "2025-01-01 00:00:29.600Z": -1.334,\n "2025-01-01 00:00:29.700Z": -1.513,\n "2025-01-01 00:00:29.800Z": -1.327,\n "2025-01-01 00:00:29.900Z": -0.776,\n "2025-01-01 00:00:30.000Z": -0.0\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2025-01-01 00:00:00.000Z": 0.0,\n "2025-01-01 00:00:00.100Z": 0.776,\n "2025-01-01 00:00:00.200Z": 1.327,\n "2025-01-01 00:00:00.300Z": 1.513,\n "2025-01-01 00:00:00.400Z": 1.334,\n "2025-01-01 00:00:00.500Z": 0.927,\n "2025-01-01 00:00:00.600Z": 0.517,\n "2025-01-01 00:00:00.700Z": 0.326,\n "2025-01-01 00:00:00.800Z": 0.494,\n "2025-01-01 00:00:00.900Z": 1.02,\n "2025-01-01 00:00:01.000Z": 1.763,\n "2025-01-01 00:00:01.100Z": 2.5,\n "2025-01-01 00:00:01.200Z": 3.005,\n "2025-01-01 00:00:01.300Z": 3.138,\n "2025-01-01 00:00:01.400Z": 2.899,\n "2025-01-01 00:00:01.500Z": 2.427,\n "2025-01-01 00:00:01.600Z": 1.945,\n "2025-01-01 00:00:01.700Z": 1.678,\n "2025-01-01 00:00:01.800Z": 1.763,\n "2025-01-01 00:00:01.900Z": 2.202,\n "2025-01-01 00:00:02.000Z": 2.853,\n "2025-01-01 00:00:02.100Z": 3.494,\n "2025-01-01 00:00:02.200Z": 3.898,\n "2025-01-01 00:00:02.300Z": 3.927,\n "2025-01-01 00:00:02.400Z": 3.582,\n "2025-01-01 00:00:02.500Z": 3.0,\n "2025-01-01 00:00:02.600Z": 2.406,\n "2025-01-01 00:00:02.700Z": 2.025,\n "2025-01-01 00:00:02.800Z": 1.996,\n "2025-01-01 00:00:02.900Z": 2.318,\n "2025-01-01 00:00:03.000Z": 2.853,\n "2025-01-01 00:00:03.100Z": 3.377,\n "2025-01-01 00:00:03.200Z": 3.666,\n "2025-01-01 00:00:03.300Z": 3.58,\n "2025-01-01 00:00:03.400Z": 3.121,\n "2025-01-01 00:00:03.500Z": 2.427,\n "2025-01-01 00:00:03.600Z": 1.724,\n "2025-01-01 00:00:03.700Z": 1.236,\n "2025-01-01 00:00:03.800Z": 1.103,\n "2025-01-01 00:00:03.900Z": 1.324,\n "2025-01-01 00:00:04.000Z": 1.763,\n "2025-01-01 00:00:04.100Z": 2.195,\n "2025-01-01 00:00:04.200Z": 2.396,\n "2025-01-01 00:00:04.300Z": 2.228,\n "2025-01-01 00:00:04.400Z": 1.692,\n "2025-01-01 00:00:04.500Z": 0.927,\n "2025-01-01 00:00:04.600Z": 0.158,\n "2025-01-01 00:00:04.700Z": -0.389,\n "2025-01-01 00:00:04.800Z": -0.575,\n "2025-01-01 00:00:04.900Z": -0.399,\n "2025-01-01 00:00:05.000Z": -0.0,\n "2025-01-01 00:00:05.100Z": 0.399,\n "2025-01-01 00:00:05.200Z": 0.575,\n "2025-01-01 00:00:05.300Z": 0.389,\n "2025-01-01 00:00:05.400Z": -0.158,\n "2025-01-01 00:00:05.500Z": -0.927,\n "2025-01-01 00:00:05.600Z": -1.692,\n "2025-01-01 00:00:05.700Z": -2.228,\n "2025-01-01 00:00:05.800Z": -2.396,\n "2025-01-01 00:00:05.900Z": -2.195,\n "2025-01-01 00:00:06.000Z": -1.763,\n "2025-01-01 00:00:06.100Z": -1.324,\n "2025-01-01 00:00:06.200Z": -1.103,\n "2025-01-01 00:00:06.300Z": -1.236,\n "2025-01-01 00:00:06.400Z": -1.724,\n "2025-01-01 00:00:06.500Z": -2.427,\n "2025-01-01 00:00:06.600Z": -3.121,\n "2025-01-01 00:00:06.700Z": -3.58,\n "2025-01-01 00:00:06.800Z": -3.666,\n "2025-01-01 00:00:06.900Z": -3.377,\n "2025-01-01 00:00:07.000Z": -2.853,\n "2025-01-01 00:00:07.100Z": -2.318,\n "2025-01-01 00:00:07.200Z": -1.996,\n "2025-01-01 00:00:07.300Z": -2.025,\n "2025-01-01 00:00:07.400Z": -2.406,\n "2025-01-01 00:00:07.500Z": -3.0,\n "2025-01-01 00:00:07.600Z": -3.582,\n "2025-01-01 00:00:07.700Z": -3.927,\n "2025-01-01 00:00:07.800Z": -3.898,\n "2025-01-01 00:00:07.900Z": -3.494,\n "2025-01-01 00:00:08.000Z": -2.853,\n "2025-01-01 00:00:08.100Z": -2.202,\n "2025-01-01 00:00:08.200Z": -1.763,\n "2025-01-01 00:00:08.300Z": -1.678,\n "2025-01-01 00:00:08.400Z": -1.945,\n "2025-01-01 00:00:08.500Z": -2.427,\n "2025-01-01 00:00:08.600Z": -2.899,\n "2025-01-01 00:00:08.700Z": -3.138,\n "2025-01-01 00:00:08.800Z": -3.005,\n "2025-01-01 00:00:08.900Z": -2.5,\n "2025-01-01 00:00:09.000Z": -1.763,\n "2025-01-01 00:00:09.100Z": -1.02,\n "2025-01-01 00:00:09.200Z": -0.494,\n "2025-01-01 00:00:09.300Z": -0.326,\n "2025-01-01 00:00:09.400Z": -0.517,\n "2025-01-01 00:00:09.500Z": -0.927,\n "2025-01-01 00:00:09.600Z": -1.334,\n "2025-01-01 00:00:09.700Z": -1.513,\n "2025-01-01 00:00:09.800Z": -1.327,\n "2025-01-01 00:00:09.900Z": -0.776,\n "2025-01-01 00:00:10.000Z": -0.0,\n "2025-01-01 00:00:10.100Z": 0.776,\n "2025-01-01 00:00:10.200Z": 1.327,\n "2025-01-01 00:00:10.300Z": 1.513,\n "2025-01-01 00:00:10.400Z": 1.334,\n "2025-01-01 00:00:10.500Z": 0.927,\n "2025-01-01 00:00:10.600Z": 0.517,\n "2025-01-01 00:00:10.700Z": 0.326,\n "2025-01-01 00:00:10.800Z": 0.494,\n "2025-01-01 00:00:10.900Z": 1.02,\n "2025-01-01 00:00:11.000Z": 1.763,\n "2025-01-01 00:00:11.100Z": 2.5,\n "2025-01-01 00:00:11.200Z": 3.005,\n "2025-01-01 00:00:11.300Z": 3.138,\n "2025-01-01 00:00:11.400Z": 2.899,\n "2025-01-01 00:00:11.500Z": 2.427,\n "2025-01-01 00:00:11.600Z": 1.945,\n "2025-01-01 00:00:11.700Z": 1.678,\n "2025-01-01 00:00:11.800Z": 1.763,\n "2025-01-01 00:00:11.900Z": 2.202,\n "2025-01-01 00:00:12.000Z": 2.853,\n "2025-01-01 00:00:12.100Z": 3.494,\n "2025-01-01 00:00:12.200Z": 3.898,\n "2025-01-01 00:00:12.300Z": 3.927,\n "2025-01-01 00:00:12.400Z": 3.582,\n "2025-01-01 00:00:12.500Z": 3.0,\n "2025-01-01 00:00:12.600Z": 2.406,\n "2025-01-01 00:00:12.700Z": 2.025,\n "2025-01-01 00:00:12.800Z": 1.996,\n "2025-01-01 00:00:12.900Z": 2.318,\n "2025-01-01 00:00:13.000Z": 2.853,\n "2025-01-01 00:00:13.100Z": 3.377,\n "2025-01-01 00:00:13.200Z": 3.666,\n "2025-01-01 00:00:13.300Z": 3.58,\n "2025-01-01 00:00:13.400Z": 3.121,\n "2025-01-01 00:00:13.500Z": 2.427,\n "2025-01-01 00:00:13.600Z": 1.724,\n "2025-01-01 00:00:13.700Z": 1.236,\n "2025-01-01 00:00:13.800Z": 1.103,\n "2025-01-01 00:00:13.900Z": 1.324,\n "2025-01-01 00:00:14.000Z": 1.763,\n "2025-01-01 00:00:14.100Z": 2.195,\n "2025-01-01 00:00:14.200Z": 2.396,\n "2025-01-01 00:00:14.300Z": 2.228,\n "2025-01-01 00:00:14.400Z": 1.692,\n "2025-01-01 00:00:14.500Z": 0.927,\n "2025-01-01 00:00:14.600Z": 0.158,\n "2025-01-01 00:00:14.700Z": -0.389,\n "2025-01-01 00:00:14.800Z": -0.575,\n "2025-01-01 00:00:14.900Z": -0.399,\n "2025-01-01 00:00:15.000Z": -0.0,\n "2025-01-01 00:00:15.100Z": 0.399,\n "2025-01-01 00:00:15.200Z": 0.575,\n "2025-01-01 00:00:15.300Z": 0.389,\n "2025-01-01 00:00:15.400Z": -0.158,\n "2025-01-01 00:00:15.500Z": -0.927,\n "2025-01-01 00:00:15.600Z": -1.692,\n "2025-01-01 00:00:15.700Z": -2.228,\n "2025-01-01 00:00:15.800Z": -2.396,\n "2025-01-01 00:00:15.900Z": -2.195,\n "2025-01-01 00:00:16.000Z": -1.763,\n "2025-01-01 00:00:16.100Z": -1.324,\n "2025-01-01 00:00:16.200Z": -1.103,\n "2025-01-01 00:00:16.300Z": -1.236,\n "2025-01-01 00:00:16.400Z": -1.724,\n "2025-01-01 00:00:16.500Z": -2.427,\n "2025-01-01 00:00:16.600Z": -3.121,\n "2025-01-01 00:00:16.700Z": -3.58,\n "2025-01-01 00:00:16.800Z": -3.666,\n "2025-01-01 00:00:16.900Z": -3.377,\n "2025-01-01 00:00:17.000Z": -2.853,\n "2025-01-01 00:00:17.100Z": -2.318,\n "2025-01-01 00:00:17.200Z": -1.996,\n "2025-01-01 00:00:17.300Z": -2.025,\n "2025-01-01 00:00:17.400Z": -2.406,\n "2025-01-01 00:00:17.500Z": -3.0,\n "2025-01-01 00:00:17.600Z": -3.582,\n "2025-01-01 00:00:17.700Z": -3.927,\n "2025-01-01 00:00:17.800Z": -3.898,\n "2025-01-01 00:00:17.900Z": -3.494,\n "2025-01-01 00:00:18.000Z": -2.853,\n "2025-01-01 00:00:18.100Z": -2.202,\n "2025-01-01 00:00:18.200Z": -1.763,\n "2025-01-01 00:00:18.300Z": -1.678,\n "2025-01-01 00:00:18.400Z": -1.945,\n "2025-01-01 00:00:18.500Z": -2.427,\n "2025-01-01 00:00:18.600Z": -2.899,\n "2025-01-01 00:00:18.700Z": -3.138,\n "2025-01-01 00:00:18.800Z": -3.005,\n "2025-01-01 00:00:18.900Z": -2.5,\n "2025-01-01 00:00:19.000Z": -1.763,\n "2025-01-01 00:00:19.100Z": -1.02,\n "2025-01-01 00:00:19.200Z": -0.494,\n "2025-01-01 00:00:19.300Z": -0.326,\n "2025-01-01 00:00:19.400Z": -0.517,\n "2025-01-01 00:00:19.500Z": -0.927,\n "2025-01-01 00:00:19.600Z": -1.334,\n "2025-01-01 00:00:19.700Z": -1.513,\n "2025-01-01 00:00:19.800Z": -1.327,\n "2025-01-01 00:00:19.900Z": -0.776,\n "2025-01-01 00:00:20.000Z": -0.0,\n "2025-01-01 00:00:20.100Z": 0.776,\n "2025-01-01 00:00:20.200Z": 1.327,\n "2025-01-01 00:00:20.300Z": 1.513,\n "2025-01-01 00:00:20.400Z": 1.334,\n "2025-01-01 00:00:20.500Z": 0.927,\n "2025-01-01 00:00:20.600Z": 0.517,\n "2025-01-01 00:00:20.700Z": 0.326,\n "2025-01-01 00:00:20.800Z": 0.494,\n "2025-01-01 00:00:20.900Z": 1.02,\n "2025-01-01 00:00:21.000Z": 1.763,\n "2025-01-01 00:00:21.100Z": 2.5,\n "2025-01-01 00:00:21.200Z": 3.005,\n "2025-01-01 00:00:21.300Z": 3.138,\n "2025-01-01 00:00:21.400Z": 2.899,\n "2025-01-01 00:00:21.500Z": 2.427,\n "2025-01-01 00:00:21.600Z": 1.945,\n "2025-01-01 00:00:21.700Z": 1.678,\n "2025-01-01 00:00:21.800Z": 1.763,\n "2025-01-01 00:00:21.900Z": 2.202,\n "2025-01-01 00:00:22.000Z": 2.853,\n "2025-01-01 00:00:22.100Z": 3.494,\n "2025-01-01 00:00:22.200Z": 3.898,\n "2025-01-01 00:00:22.300Z": 3.927,\n "2025-01-01 00:00:22.400Z": 3.582,\n "2025-01-01 00:00:22.500Z": 3.0,\n "2025-01-01 00:00:22.600Z": 2.406,\n "2025-01-01 00:00:22.700Z": 2.025,\n "2025-01-01 00:00:22.800Z": 1.996,\n "2025-01-01 00:00:22.900Z": 2.318,\n "2025-01-01 00:00:23.000Z": 2.853,\n "2025-01-01 00:00:23.100Z": 3.377,\n "2025-01-01 00:00:23.200Z": 3.666,\n "2025-01-01 00:00:23.300Z": 3.58,\n "2025-01-01 00:00:23.400Z": 3.121,\n "2025-01-01 00:00:23.500Z": 2.427,\n "2025-01-01 00:00:23.600Z": 1.724,\n "2025-01-01 00:00:23.700Z": 1.236,\n "2025-01-01 00:00:23.800Z": 1.103,\n "2025-01-01 00:00:23.900Z": 1.324,\n "2025-01-01 00:00:24.000Z": 1.763,\n "2025-01-01 00:00:24.100Z": 2.195,\n "2025-01-01 00:00:24.200Z": 2.396,\n "2025-01-01 00:00:24.300Z": 2.228,\n "2025-01-01 00:00:24.400Z": 1.692,\n "2025-01-01 00:00:24.500Z": 0.927,\n "2025-01-01 00:00:24.600Z": 0.158,\n "2025-01-01 00:00:24.700Z": -0.389,\n "2025-01-01 00:00:24.800Z": -0.575,\n "2025-01-01 00:00:24.900Z": -0.399,\n "2025-01-01 00:00:25.000Z": 0.0,\n "2025-01-01 00:00:25.100Z": 0.399,\n "2025-01-01 00:00:25.200Z": 0.575,\n "2025-01-01 00:00:25.300Z": 0.389,\n "2025-01-01 00:00:25.400Z": -0.158,\n "2025-01-01 00:00:25.500Z": -0.927,\n "2025-01-01 00:00:25.600Z": -1.692,\n "2025-01-01 00:00:25.700Z": -2.228,\n "2025-01-01 00:00:25.800Z": -2.396,\n "2025-01-01 00:00:25.900Z": -2.195,\n "2025-01-01 00:00:26.000Z": -1.763,\n "2025-01-01 00:00:26.100Z": -1.324,\n "2025-01-01 00:00:26.200Z": -1.103,\n "2025-01-01 00:00:26.300Z": -1.236,\n "2025-01-01 00:00:26.400Z": -1.724,\n "2025-01-01 00:00:26.500Z": -2.427,\n "2025-01-01 00:00:26.600Z": -3.121,\n "2025-01-01 00:00:26.700Z": -3.58,\n "2025-01-01 00:00:26.800Z": -3.666,\n "2025-01-01 00:00:26.900Z": -3.377,\n "2025-01-01 00:00:27.000Z": -2.853,\n "2025-01-01 00:00:27.100Z": -2.318,\n "2025-01-01 00:00:27.200Z": -1.996,\n "2025-01-01 00:00:27.300Z": -2.025,\n "2025-01-01 00:00:27.400Z": -2.406,\n "2025-01-01 00:00:27.500Z": -3.0,\n "2025-01-01 00:00:27.600Z": -3.582,\n "2025-01-01 00:00:27.700Z": -3.927,\n "2025-01-01 00:00:27.800Z": -3.898,\n "2025-01-01 00:00:27.900Z": -3.494,\n "2025-01-01 00:00:28.000Z": -2.853,\n "2025-01-01 00:00:28.100Z": -2.202,\n "2025-01-01 00:00:28.200Z": -1.763,\n "2025-01-01 00:00:28.300Z": -1.678,\n "2025-01-01 00:00:28.400Z": -1.945,\n "2025-01-01 00:00:28.500Z": -2.427,\n "2025-01-01 00:00:28.600Z": -2.899,\n "2025-01-01 00:00:28.700Z": -3.138,\n "2025-01-01 00:00:28.800Z": -3.005,\n "2025-01-01 00:00:28.900Z": -2.5,\n "2025-01-01 00:00:29.000Z": -1.763,\n "2025-01-01 00:00:29.100Z": -1.02,\n "2025-01-01 00:00:29.200Z": -0.494,\n "2025-01-01 00:00:29.300Z": -0.326,\n "2025-01-01 00:00:29.400Z": -0.517,\n "2025-01-01 00:00:29.500Z": -0.927,\n "2025-01-01 00:00:29.600Z": -1.334,\n "2025-01-01 00:00:29.700Z": -1.513,\n "2025-01-01 00:00:29.800Z": -1.327,\n "2025-01-01 00:00:29.900Z": -0.776,\n "2025-01-01 00:00:30.000Z": -0.0\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/handle_gaps_and_missing_data_in_time_series_100_06b9bb8f-513f-4304-8f86-2bf6d5ba32f1.py b/runtime/transformations/components/time-series-base-components/handle_gaps_and_missing_data_in_time_series_100_06b9bb8f-513f-4304-8f86-2bf6d5ba32f1.py new file mode 100644 index 000000000..ea3ca1daf --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/handle_gaps_and_missing_data_in_time_series_100_06b9bb8f-513f-4304-8f86-2bf6d5ba32f1.py @@ -0,0 +1,672 @@ +"""Documentation for Handle Gaps and Missing Data in Time Series + +# Handle Gaps and Missing Data in Time Series + +## Description +Single-point component to detect gaps (missing values and missing timestamps), +optionally fill them, and return a corrected series. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. Index must be datetime, values numeric. + Optional metadata in `timeseries.attrs` is supported: + `ref_interval_start_timestamp` / `from` and + `ref_interval_end_timestamp` / `to`. +- **mode** (String, default value: "fill"): + One of "fill", "flag", "drop". +- **method** (String, default value: "time"): + Filling method. One of "time", "linear", "ffill", "bfill", "constant". +- **fill_direction** (String, default value: "both"): + Direction in which filling is allowed ("forward", "backward", "both"). +- **max_gap_duration** (String, default value: null): + Maximum size of an inner gap that may still be filled, for example `15min`, + `1h`, or `1D`. Larger inner gaps are left as missing values. If not set, + the component uses about six typical sampling intervals. +- **constant_value** (Float, default value: 0): + Constant value used when method="constant". +- **resample_to** (String, default value: null): + Optional target frequency (e.g. "10min") to create a regular grid. + Missing timestamps on the grid are treated as gaps. If `resample_to` is not + set, the component may still derive an internal frequency when this is + needed to apply edge handling consistently. + +## Outputs +- **corrected_timeseries** (Pandas Series): + The resulting series (filled/flagged/dropped depending on mode). + +## Details +1. Sorts the input by time and removes duplicate timestamps (keeps the mean). +2. Optionally resamples to a regular grid to make missing timestamps visible. +3. Detects gaps as NaN values and as missing timestamps on the grid. +4. If `resample_to` is set, the component first creates that regular grid. +5. If `resample_to` is not set, the component may still derive an internal + frequency when this is needed to apply reference interval metadata and edge + handling consistently. +6. The grid can optionally be extended/restricted to the reference interval from + metadata (`ref_interval_start_timestamp`/`from`, + `ref_interval_end_timestamp`/`to`). + If metadata provides `ref_data_frequency` (and optionally + `ref_data_frequency_offset`), these values are preferred for grid building. + If interval boundaries are not aligned to the detected grid, boundaries are + snapped to the nearest inner grid points while preserving the original + timestamp phase of the series. +7. Gaps inside the measured data are treated with the selected fill method and + the configured maximum gap duration. +8. Missing points before the first real value are always filled with that first value. +9. Missing points after the last real value are never filled. +10. This edge behavior is independent of how inner gaps are handled. +11. Returns the processed series. + +## Example +```json +{ + "timeseries": { + "2026-01-12T00:00:00Z": 10, + "2026-01-12T00:07:00Z": 12, + "2026-01-12T00:13:00Z": 13, + "2026-01-12T00:18:00Z": 14, + "2026-01-12T00:22:00Z": 15, + "2026-01-12T00:25:00Z": 15, + "2026-01-12T00:32:00Z": 15, + "2026-01-12T00:38:00Z": 14, + "2026-01-12T00:43:00Z": 13, + "2026-01-12T00:47:00Z": 12, + "2026-01-12T00:50:00Z": 11, + "2026-01-12T00:57:00Z": 10, + "2026-01-12T01:03:00Z": null, + "2026-01-12T01:08:00Z": 9, + "2026-01-12T01:12:00Z": 9, + "2026-01-12T01:15:00Z": 10, + "2026-01-12T01:22:00Z": 11, + "2026-01-12T01:28:00Z": 12, + "2026-01-12T01:33:00Z": 14, + "2026-01-12T01:37:00Z": 15, + "2026-01-12T01:40:00Z": 17, + "2026-01-12T01:47:00Z": 18, + "2026-01-12T01:53:00Z": 18, + "2026-01-12T01:58:00Z": 19, + "2026-01-12T02:02:00Z": 18, + "2026-01-12T02:05:00Z": 18, + "2026-01-12T02:12:00Z": 16, + "2026-01-12T02:18:00Z": 15, + "2026-01-12T02:23:00Z": 14, + "2026-01-12T02:27:00Z": 13, + "2026-01-12T02:30:00Z": null, + "2026-01-12T02:37:00Z": null, + "2026-01-12T02:43:00Z": null, + "2026-01-12T02:48:00Z": 13, + "2026-01-12T02:52:00Z": 14, + "2026-01-12T02:55:00Z": 15, + "2026-01-12T03:02:00Z": 16, + "2026-01-12T03:08:00Z": 18, + "2026-01-12T03:13:00Z": 19, + "2026-01-12T03:17:00Z": 20, + "2026-01-12T03:20:00Z": 21, + "2026-01-12T03:27:00Z": 21, + "2026-01-12T03:33:00Z": 20, + "2026-01-12T03:38:00Z": 20, + "2026-01-12T03:42:00Z": 19, + "2026-01-12T03:45:00Z": 17, + "2026-01-12T03:52:00Z": 16, + "2026-01-12T03:58:00Z": 15, + "2026-01-12T04:03:00Z": 14, + "2026-01-12T04:07:00Z": 13, + "2026-01-12T04:10:00Z": 13, + "2026-01-12T04:17:00Z": 14, + "2026-01-12T04:23:00Z": 15, + "2026-01-12T04:28:00Z": 16, + "2026-01-12T04:32:00Z": null, + "2026-01-12T04:35:00Z": null, + "2026-01-12T04:42:00Z": null, + "2026-01-12T04:48:00Z": null, + "2026-01-12T04:53:00Z": null, + "2026-01-12T04:57:00Z": null, + "2026-01-12T05:00:00Z": 21, + "2026-01-12T05:07:00Z": 20, + "2026-01-12T05:13:00Z": 19 +}, + "max_gap_duration": "20min" +} +``` +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + + +def validate_inputs( + series: pd.Series, + mode: str, + method: str, + fill_direction: str, + max_gap_duration: str | None, + constant_value: float, + resample_to: str | None, +) -> None: + if not isinstance(series, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(series.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if mode not in {"fill", "flag", "drop"}: + raise ComponentInputValidationException( + f"mode must be one of 'fill', 'flag', 'drop', got '{mode}'", + error_code="422", + invalid_component_inputs=["mode"], + ) + if method not in {"time", "linear", "ffill", "bfill", "constant"}: + raise ComponentInputValidationException( + f"method must be one of 'time', 'linear', 'ffill', 'bfill', 'constant', got '{method}'", + error_code="422", + invalid_component_inputs=["method"], + ) + if fill_direction not in {"forward", "backward", "both"}: + raise ComponentInputValidationException( + "fill_direction must be one of 'forward', 'backward', 'both'", + error_code="422", + invalid_component_inputs=["fill_direction"], + ) + if max_gap_duration is not None and not isinstance(max_gap_duration, str): + raise ComponentInputValidationException( + "max_gap_duration must be a duration string like '15min' or null", + error_code="422", + invalid_component_inputs=["max_gap_duration"], + ) + if max_gap_duration is not None: + try: + parsed_gap_duration = pd.to_timedelta(max_gap_duration) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "max_gap_duration must be a valid duration string like '15min'", + error_code="422", + invalid_component_inputs=["max_gap_duration"], + ) from exc + if parsed_gap_duration <= pd.Timedelta(0): + raise ComponentInputValidationException( + "max_gap_duration must be positive", + error_code="422", + invalid_component_inputs=["max_gap_duration"], + ) + if mode == "fill" and method == "constant": + if not isinstance(constant_value, (int, float)): + raise ComponentInputValidationException( + "constant_value must be a number", + error_code="422", + invalid_component_inputs=["constant_value"], + ) + if not np.isfinite(float(constant_value)): + raise ComponentInputValidationException( + "constant_value must be a finite number", + error_code="422", + invalid_component_inputs=["constant_value"], + ) + if resample_to is not None and not isinstance(resample_to, str): + raise ComponentInputValidationException( + "resample_to must be a frequency string like '5min' or null", + error_code="422", + invalid_component_inputs=["resample_to"], + ) + + +def gap_lengths(mask: pd.Series) -> pd.Series: + """Return gap length for each position in a boolean gap mask.""" + group = (mask != mask.shift()).cumsum() + return mask.groupby(group).transform("sum") + + +def split_gap_masks(series: pd.Series) -> tuple[pd.Series, pd.Series, pd.Series]: + missing_mask = series.isna() + left_edge_mask = pd.Series(False, index=series.index) + inner_gap_mask = pd.Series(False, index=series.index) + right_edge_mask = pd.Series(False, index=series.index) + + if not missing_mask.any(): + return left_edge_mask, inner_gap_mask, right_edge_mask + + first_valid = series.first_valid_index() + last_valid = series.last_valid_index() + + if first_valid is None or last_valid is None: + return left_edge_mask, inner_gap_mask, right_edge_mask + + left_edge_mask = missing_mask & (series.index < first_valid) + right_edge_mask = missing_mask & (series.index > last_valid) + inner_gap_mask = missing_mask & ~(left_edge_mask | right_edge_mask) + return left_edge_mask, inner_gap_mask, right_edge_mask + + +def infer_typical_step(series: pd.Series) -> pd.Timedelta | None: + diffs = series.index.to_series().diff().dropna() + positive_diffs = diffs[diffs > pd.Timedelta(0)] + if positive_diffs.empty: + return None + return positive_diffs.median() + + +def resolve_max_gap_duration( + series: pd.Series, max_gap_duration: str | None +) -> pd.Timedelta | None: + if max_gap_duration is not None: + return pd.to_timedelta(max_gap_duration) + + typical_step = infer_typical_step(series) + if typical_step is None: + return None + return 6 * typical_step + + +def get_reference_interval_from_series_attrs( + series: pd.Series, +) -> tuple[pd.Timestamp | None, pd.Timestamp | None]: + """Read optional interval boundaries from series metadata. + + Supported keys: + - start: ``ref_interval_start_timestamp`` or ``from`` + - end: ``ref_interval_end_timestamp`` or ``to`` + """ + + attrs = series.attrs if isinstance(series.attrs, dict) else {} + dataset_metadata = attrs.get("dataset_metadata") + + start_raw = None + end_raw = None + if isinstance(dataset_metadata, dict): + start_raw = dataset_metadata.get( + "ref_interval_start_timestamp", dataset_metadata.get("from") + ) + end_raw = dataset_metadata.get("ref_interval_end_timestamp", dataset_metadata.get("to")) + + if start_raw is None: + start_raw = attrs.get("ref_interval_start_timestamp", attrs.get("from")) + if end_raw is None: + end_raw = attrs.get("ref_interval_end_timestamp", attrs.get("to")) + + start_ts: pd.Timestamp | None = None + end_ts: pd.Timestamp | None = None + + if start_raw is not None: + try: + start_ts = pd.Timestamp(start_raw) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "timeseries metadata field 'ref_interval_start_timestamp' (or 'from') is not a valid timestamp", + error_code="422", + invalid_component_inputs=["timeseries"], + ) from exc + + if end_raw is not None: + try: + end_ts = pd.Timestamp(end_raw) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "timeseries metadata field 'ref_interval_end_timestamp' (or 'to') is not a valid timestamp", + error_code="422", + invalid_component_inputs=["timeseries"], + ) from exc + + if start_ts is not None and end_ts is not None and start_ts > end_ts: + raise ComponentInputValidationException( + "timeseries metadata interval is invalid: start is after end", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + return start_ts, end_ts + + +def get_reference_frequency_from_series_attrs(series: pd.Series) -> str | None: + """Read optional reference frequency from series metadata.""" + + attrs = series.attrs if isinstance(series.attrs, dict) else {} + dataset_metadata = attrs.get("dataset_metadata") + + freq_raw = None + if isinstance(dataset_metadata, dict): + freq_raw = dataset_metadata.get("ref_data_frequency") + if freq_raw is None: + freq_raw = attrs.get("ref_data_frequency") + + if freq_raw is None: + return None + if not isinstance(freq_raw, str): + raise ComponentInputValidationException( + "timeseries metadata field 'ref_data_frequency' must be a frequency string", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + return freq_raw + + +def get_reference_frequency_offset_from_series_attrs(series: pd.Series) -> str | None: + """Read optional reference frequency offset from series metadata.""" + + attrs = series.attrs if isinstance(series.attrs, dict) else {} + dataset_metadata = attrs.get("dataset_metadata") + + offset_raw = None + if isinstance(dataset_metadata, dict): + offset_raw = dataset_metadata.get("ref_data_frequency_offset") + if offset_raw is None: + offset_raw = attrs.get("ref_data_frequency_offset") + + if offset_raw is None: + return None + if not isinstance(offset_raw, str): + raise ComponentInputValidationException( + "timeseries metadata field 'ref_data_frequency_offset' must be a duration string", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + return offset_raw + + +def build_regular_grid_from_frequency( + ordered: pd.Series, + frequency: str | pd.Timedelta, +) -> pd.Series: + full_index = pd.date_range(start=ordered.index.min(), end=ordered.index.max(), freq=frequency) + return ordered.reindex(full_index) + + +def resolve_frequency_for_window( + ordered: pd.Series, + resample_to: str | None, + reference_frequency: str | None, +) -> tuple[pd.Series, str | pd.Timedelta | None]: + resample_value = resample_to + if resample_value is False or resample_value == "": + resample_value = None + + if resample_value: + try: + ordered = build_regular_grid_from_frequency(ordered, resample_value) + except (ValueError, TypeError) as exc: + raise ComponentInputValidationException( + f"resample_to could not be parsed as frequency: {resample_value}", + error_code="422", + invalid_component_inputs=["resample_to"], + ) from exc + return ordered, resample_value + + if reference_frequency and len(ordered.index) > 1: + try: + ref_freq_delta = pd.to_timedelta(reference_frequency) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "timeseries metadata field 'ref_data_frequency' is not a valid frequency", + error_code="422", + invalid_component_inputs=["timeseries"], + ) from exc + if ref_freq_delta <= pd.Timedelta(0): + raise ComponentInputValidationException( + "timeseries metadata field 'ref_data_frequency' must be positive", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + ordered = build_regular_grid_from_frequency(ordered, ref_freq_delta) + return ordered, ref_freq_delta + + return ordered, None + + +def resolve_frequency_delta_for_window( + ordered: pd.Series, + frequency_for_window: str | pd.Timedelta | None, +) -> pd.Timedelta: + if frequency_for_window is None: + diffs = ordered.index.to_series().diff().dropna() + positive_diffs = diffs[diffs > pd.Timedelta(0)] + if positive_diffs.empty: + raise ComponentInputValidationException( + "Cannot apply metadata interval without a detectable positive frequency", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + frequency_for_window = positive_diffs.median() + + try: + freq_delta = pd.to_timedelta(frequency_for_window) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "Cannot apply metadata interval because frequency is not a fixed timedelta", + error_code="422", + invalid_component_inputs=["timeseries"], + ) from exc + if freq_delta <= pd.Timedelta(0): + raise ComponentInputValidationException( + "Cannot apply metadata interval because frequency must be positive", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + return freq_delta + + +def apply_reference_window_to_series( + ordered: pd.Series, + window_start: pd.Timestamp | None, + window_end: pd.Timestamp | None, + reference_frequency_offset: str | None, + frequency_for_window: str | pd.Timedelta | None, +) -> pd.Series: + if window_start is None and window_end is None: + return ordered + if ordered.empty: + raise ComponentInputValidationException( + "Cannot apply metadata interval to an empty timeseries", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + freq_delta = resolve_frequency_delta_for_window(ordered, frequency_for_window) + + target_start = window_start if window_start is not None else ordered.index.min() + target_end = window_end if window_end is not None else ordered.index.max() + if target_start > target_end: + raise ComponentInputValidationException( + "Metadata interval start must be before or equal to end", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + # Keep the original timestamp phase (e.g. full hour) across the whole window. + anchor = ordered.index.min() + if reference_frequency_offset is not None: + try: + offset_delta = pd.to_timedelta(reference_frequency_offset) + except (TypeError, ValueError) as exc: + raise ComponentInputValidationException( + "timeseries metadata field 'ref_data_frequency_offset' is not a valid duration", + error_code="422", + invalid_component_inputs=["timeseries"], + ) from exc + offset_mod = offset_delta % freq_delta + epoch_anchor = pd.Timestamp("1970-01-01", tz=ordered.index.min().tz) + anchor = epoch_anchor + offset_mod + + start_steps = int(np.ceil((target_start - anchor) / freq_delta)) + end_steps = int(np.floor((target_end - anchor) / freq_delta)) + aligned_start = anchor + start_steps * freq_delta + aligned_end = anchor + end_steps * freq_delta + + if aligned_start > aligned_end: + return ordered.iloc[0:0] + + full_window_index = pd.date_range(start=aligned_start, end=aligned_end, freq=freq_delta) + return ordered.reindex(full_window_index) + + +def prepare_series( + series: pd.Series, + resample_to: str | None, + reference_frequency: str | None = None, + reference_frequency_offset: str | None = None, + window_start: pd.Timestamp | None = None, + window_end: pd.Timestamp | None = None, +) -> pd.Series: + ordered = series.sort_index() + if not ordered.index.is_unique: + ordered = ordered.groupby(level=0).mean() + ordered, frequency_for_window = resolve_frequency_for_window( + ordered, + resample_to, + reference_frequency, + ) + return apply_reference_window_to_series( + ordered, + window_start=window_start, + window_end=window_end, + reference_frequency_offset=reference_frequency_offset, + frequency_for_window=frequency_for_window, + ) + + +def fill_series( + series: pd.Series, + fillable_mask: pd.Series, + method: str, + fill_direction: str, + constant_value: float, +) -> pd.Series: + if method == "constant": + filled = series.copy() + filled.loc[fillable_mask] = constant_value + return filled + + if method in {"ffill", "bfill"}: + filled = series.ffill() if method == "ffill" else series.bfill() + else: + filled = series.interpolate(method=method, limit_direction=fill_direction) + + # Restore non-fillable missing points + filled.loc[~fillable_mask & series.isna()] = np.nan + return filled + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "mode": {"data_type": "STRING", "default_value": "fill"}, + "method": {"data_type": "STRING", "default_value": "time"}, + "fill_direction": {"data_type": "STRING", "default_value": "both"}, + "max_gap_duration": {"data_type": "STRING", "default_value": None}, + "constant_value": {"data_type": "FLOAT", "default_value": 0.0}, + "resample_to": {"data_type": "STRING", "default_value": None}, + }, + "outputs": { + "corrected_timeseries": {"data_type": "SERIES"}, + }, + "name": "Handle Gaps and Missing Data in Time Series", + "category": "Time Series Base Components", + "description": "Detect and optionally fill gaps in time series.", + "version_tag": "1.0.0", + "id": "06b9bb8f-513f-4304-8f86-2bf6d5ba32f1", + "revision_group_id": "2c3fb56e-5288-42a0-aa91-4417b29fb2af", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + mode="fill", + method="time", + fill_direction="both", + max_gap_duration=None, + constant_value=0.0, + resample_to=None, +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + validate_inputs( + timeseries, + mode, + method, + fill_direction, + max_gap_duration, + constant_value, + resample_to, + ) + ref_interval_start, ref_interval_end = get_reference_interval_from_series_attrs(timeseries) + ref_data_frequency = get_reference_frequency_from_series_attrs(timeseries) + ref_data_frequency_offset = get_reference_frequency_offset_from_series_attrs(timeseries) + series = prepare_series( + timeseries, + resample_to, + reference_frequency=ref_data_frequency, + reference_frequency_offset=ref_data_frequency_offset, + window_start=ref_interval_start, + window_end=ref_interval_end, + ) + + left_edge_mask, inner_gap_mask, right_edge_mask = split_gap_masks(series) + gap_lengths_values = gap_lengths(inner_gap_mask) + fillable_mask = inner_gap_mask.copy() + resolved_max_gap_duration = resolve_max_gap_duration(series, max_gap_duration) + if resolved_max_gap_duration is not None: + typical_step = infer_typical_step(series) + if typical_step is not None: + gap_durations = gap_lengths_values * typical_step + fillable_mask &= gap_durations <= resolved_max_gap_duration + + processed = series.copy() + first_valid = series.first_valid_index() + if first_valid is not None and left_edge_mask.any(): + processed.loc[left_edge_mask] = float(series.loc[first_valid]) + processed.loc[right_edge_mask] = np.nan + + if mode == "fill": + processed = fill_series( + processed, + fillable_mask, + method, + fill_direction, + constant_value, + ) + processed.loc[right_edge_mask] = np.nan + elif mode == "drop": + processed = processed.dropna() + + return { + "corrected_timeseries": processed, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-01-12T00:00:00Z": 10,\n "2026-01-12T00:07:00Z": 12,\n "2026-01-12T00:13:00Z": 13,\n "2026-01-12T00:18:00Z": 14,\n "2026-01-12T00:22:00Z": 15,\n "2026-01-12T00:25:00Z": 15,\n "2026-01-12T00:32:00Z": 15,\n "2026-01-12T00:38:00Z": 14,\n "2026-01-12T00:43:00Z": 13,\n "2026-01-12T00:47:00Z": 12,\n "2026-01-12T00:50:00Z": 11,\n "2026-01-12T00:57:00Z": 10,\n "2026-01-12T01:03:00Z": null,\n "2026-01-12T01:08:00Z": 9,\n "2026-01-12T01:12:00Z": 9,\n "2026-01-12T01:15:00Z": 10,\n "2026-01-12T01:22:00Z": 11,\n "2026-01-12T01:28:00Z": 12,\n "2026-01-12T01:33:00Z": 14,\n "2026-01-12T01:37:00Z": 15,\n "2026-01-12T01:40:00Z": 17,\n "2026-01-12T01:47:00Z": 18,\n "2026-01-12T01:53:00Z": 18,\n "2026-01-12T01:58:00Z": 19,\n "2026-01-12T02:02:00Z": 18,\n "2026-01-12T02:05:00Z": 18,\n "2026-01-12T02:12:00Z": 16,\n "2026-01-12T02:18:00Z": 15,\n "2026-01-12T02:23:00Z": 14,\n "2026-01-12T02:27:00Z": 13,\n "2026-01-12T02:30:00Z": null,\n "2026-01-12T02:37:00Z": null,\n "2026-01-12T02:43:00Z": null,\n "2026-01-12T02:48:00Z": 13,\n "2026-01-12T02:52:00Z": 14,\n "2026-01-12T02:55:00Z": 15,\n "2026-01-12T03:02:00Z": 16,\n "2026-01-12T03:08:00Z": 18,\n "2026-01-12T03:13:00Z": 19,\n "2026-01-12T03:17:00Z": 20,\n "2026-01-12T03:20:00Z": 21,\n "2026-01-12T03:27:00Z": 21,\n "2026-01-12T03:33:00Z": 20,\n "2026-01-12T03:38:00Z": 20,\n "2026-01-12T03:42:00Z": 19,\n "2026-01-12T03:45:00Z": 17,\n "2026-01-12T03:52:00Z": 16,\n "2026-01-12T03:58:00Z": 15,\n "2026-01-12T04:03:00Z": 14,\n "2026-01-12T04:07:00Z": 13,\n "2026-01-12T04:10:00Z": 13,\n "2026-01-12T04:17:00Z": 14,\n "2026-01-12T04:23:00Z": 15,\n "2026-01-12T04:28:00Z": 16,\n "2026-01-12T04:32:00Z": null,\n "2026-01-12T04:35:00Z": null,\n "2026-01-12T04:42:00Z": null,\n "2026-01-12T04:48:00Z": null,\n "2026-01-12T04:53:00Z": null,\n "2026-01-12T04:57:00Z": null,\n "2026-01-12T05:00:00Z": 21,\n "2026-01-12T05:07:00Z": 20,\n "2026-01-12T05:13:00Z": 19,\n "2026-01-12T05:18:00Z": 17,\n "2026-01-12T05:22:00Z": 16,\n "2026-01-12T05:25:00Z": 15,\n "2026-01-12T05:32:00Z": 14,\n "2026-01-12T05:38:00Z": 13,\n "2026-01-12T05:43:00Z": 13,\n "2026-01-12T05:47:00Z": 14,\n "2026-01-12T05:50:00Z": 15,\n "2026-01-12T05:57:00Z": 16,\n "2026-01-12T06:03:00Z": 17,\n "2026-01-12T06:08:00Z": 18,\n "2026-01-12T06:12:00Z": 19,\n "2026-01-12T06:15:00Z": 20,\n "2026-01-12T06:22:00Z": 21,\n "2026-01-12T06:28:00Z": 21,\n "2026-01-12T06:33:00Z": 20,\n "2026-01-12T06:37:00Z": 19,\n "2026-01-12T06:40:00Z": 18,\n "2026-01-12T06:47:00Z": 16,\n "2026-01-12T06:53:00Z": 15,\n "2026-01-12T06:58:00Z": 14,\n "2026-01-12T07:02:00Z": 13,\n "2026-01-12T07:05:00Z": 12,\n "2026-01-12T07:12:00Z": 12,\n "2026-01-12T07:18:00Z": 12,\n "2026-01-12T07:23:00Z": 13,\n "2026-01-12T07:27:00Z": 14,\n "2026-01-12T07:30:00Z": 15,\n "2026-01-12T07:37:00Z": 17,\n "2026-01-12T07:43:00Z": 18,\n "2026-01-12T07:48:00Z": 19,\n "2026-01-12T07:52:00Z": 19,\n "2026-01-12T07:55:00Z": 19,\n "2026-01-12T08:02:00Z": null,\n "2026-01-12T08:08:00Z": null,\n "2026-01-12T08:13:00Z": null,\n "2026-01-12T08:17:00Z": null,\n "2026-01-12T08:20:00Z": null,\n "2026-01-12T08:27:00Z": null,\n "2026-01-12T08:33:00Z": null,\n "2026-01-12T08:38:00Z": null,\n "2026-01-12T08:42:00Z": null,\n "2026-01-12T08:45:00Z": null,\n "2026-01-12T08:52:00Z": null,\n "2026-01-12T08:58:00Z": null,\n "2026-01-12T09:03:00Z": null,\n "2026-01-12T09:07:00Z": null,\n "2026-01-12T09:10:00Z": null,\n "2026-01-12T09:17:00Z": null,\n "2026-01-12T09:23:00Z": null,\n "2026-01-12T09:28:00Z": null,\n "2026-01-12T09:32:00Z": null,\n "2026-01-12T09:35:00Z": null,\n "2026-01-12T09:42:00Z": null,\n "2026-01-12T09:48:00Z": null,\n "2026-01-12T09:53:00Z": null,\n "2026-01-12T09:57:00Z": null,\n "2026-01-12T10:00:00Z": null,\n "2026-01-12T10:07:00Z": null,\n "2026-01-12T10:13:00Z": null,\n "2026-01-12T10:18:00Z": null,\n "2026-01-12T10:22:00Z": null,\n "2026-01-12T10:25:00Z": null,\n "2026-01-12T10:32:00Z": 12,\n "2026-01-12T10:38:00Z": 13,\n "2026-01-12T10:43:00Z": 14,\n "2026-01-12T10:47:00Z": 15,\n "2026-01-12T10:50:00Z": 15,\n "2026-01-12T10:57:00Z": 15,\n "2026-01-12T11:03:00Z": 15,\n "2026-01-12T11:08:00Z": 14,\n "2026-01-12T11:12:00Z": 12,\n "2026-01-12T11:15:00Z": 11,\n "2026-01-12T11:22:00Z": 9,\n "2026-01-12T11:28:00Z": 8,\n "2026-01-12T11:33:00Z": 7,\n "2026-01-12T11:37:00Z": 6,\n "2026-01-12T11:40:00Z": null,\n "2026-01-12T11:47:00Z": null,\n "2026-01-12T11:53:00Z": 8,\n "2026-01-12T11:58:00Z": 9,\n "2026-01-12T12:02:00Z": 10,\n "2026-01-12T12:05:00Z": 11,\n "2026-01-12T12:12:00Z": 12,\n "2026-01-12T12:18:00Z": 13,\n "2026-01-12T12:23:00Z": 14,\n "2026-01-12T12:27:00Z": 14,\n "2026-01-12T12:30:00Z": 13,\n "2026-01-12T12:37:00Z": 12,\n "2026-01-12T12:43:00Z": 11,\n "2026-01-12T12:48:00Z": 9,\n "2026-01-12T12:52:00Z": 8,\n "2026-01-12T12:55:00Z": 6,\n "2026-01-12T13:02:00Z": 5,\n "2026-01-12T13:08:00Z": 5,\n "2026-01-12T13:13:00Z": 5,\n "2026-01-12T13:17:00Z": 5,\n "2026-01-12T13:20:00Z": 6,\n "2026-01-12T13:27:00Z": 7,\n "2026-01-12T13:33:00Z": 8,\n "2026-01-12T13:38:00Z": 10,\n "2026-01-12T13:42:00Z": 11,\n "2026-01-12T13:45:00Z": null,\n "2026-01-12T13:52:00Z": null,\n "2026-01-12T13:58:00Z": null,\n "2026-01-12T14:03:00Z": null,\n "2026-01-12T14:07:00Z": null,\n "2026-01-12T14:10:00Z": null,\n "2026-01-12T14:17:00Z": null,\n "2026-01-12T14:23:00Z": null,\n "2026-01-12T14:28:00Z": 5,\n "2026-01-12T14:32:00Z": 4,\n "2026-01-12T14:35:00Z": 3,\n "2026-01-12T14:42:00Z": 3,\n "2026-01-12T14:48:00Z": 3,\n "2026-01-12T14:53:00Z": 4,\n "2026-01-12T14:57:00Z": 5,\n "2026-01-12T15:00:00Z": 7,\n "2026-01-12T15:07:00Z": 8,\n "2026-01-12T15:13:00Z": 9,\n "2026-01-12T15:18:00Z": 10,\n "2026-01-12T15:22:00Z": 10,\n "2026-01-12T15:25:00Z": 10,\n "2026-01-12T15:32:00Z": 9,\n "2026-01-12T15:38:00Z": 8,\n "2026-01-12T15:43:00Z": 7,\n "2026-01-12T15:47:00Z": 6,\n "2026-01-12T15:50:00Z": 4,\n "2026-01-12T15:57:00Z": 3,\n "2026-01-12T16:03:00Z": 2,\n "2026-01-12T16:08:00Z": 1,\n "2026-01-12T16:12:00Z": 1,\n "2026-01-12T16:15:00Z": 1,\n "2026-01-12T16:22:00Z": 2,\n "2026-01-12T16:28:00Z": 3,\n "2026-01-12T16:33:00Z": 5,\n "2026-01-12T16:37:00Z": 6,\n "2026-01-12T16:40:00Z": null,\n "2026-01-12T16:47:00Z": null,\n "2026-01-12T16:53:00Z": null,\n "2026-01-12T16:58:00Z": null,\n "2026-01-12T17:02:00Z": 7,\n "2026-01-12T17:05:00Z": 6,\n "2026-01-12T17:12:00Z": 5,\n "2026-01-12T17:18:00Z": 4,\n "2026-01-12T17:23:00Z": 2,\n "2026-01-12T17:27:00Z": 1,\n "2026-01-12T17:30:00Z": 0,\n "2026-01-12T17:37:00Z": -1,\n "2026-01-12T17:43:00Z": -1,\n "2026-01-12T17:48:00Z": 0,\n "2026-01-12T17:52:00Z": 1,\n "2026-01-12T17:55:00Z": 2,\n "2026-01-12T18:02:00Z": 3,\n "2026-01-12T18:08:00Z": 4,\n "2026-01-12T18:13:00Z": 5,\n "2026-01-12T18:17:00Z": 6,\n "2026-01-12T18:20:00Z": null,\n "2026-01-12T18:27:00Z": null,\n "2026-01-12T18:33:00Z": null,\n "2026-01-12T18:38:00Z": null,\n "2026-01-12T18:42:00Z": null,\n "2026-01-12T18:45:00Z": null,\n "2026-01-12T18:52:00Z": null,\n "2026-01-12T18:58:00Z": null,\n "2026-01-12T19:03:00Z": null,\n "2026-01-12T19:07:00Z": -1,\n "2026-01-12T19:10:00Z": -1,\n "2026-01-12T19:17:00Z": -1,\n "2026-01-12T19:23:00Z": 0,\n "2026-01-12T19:28:00Z": 1,\n "2026-01-12T19:32:00Z": 2,\n "2026-01-12T19:35:00Z": 4,\n "2026-01-12T19:42:00Z": null,\n "2026-01-12T19:48:00Z": null,\n "2026-01-12T19:53:00Z": null,\n "2026-01-12T19:57:00Z": null,\n "2026-01-12T20:00:00Z": null,\n "2026-01-12T20:07:00Z": 5,\n "2026-01-12T20:13:00Z": 4,\n "2026-01-12T20:18:00Z": 3,\n "2026-01-12T20:22:00Z": 1,\n "2026-01-12T20:25:00Z": 0,\n "2026-01-12T20:32:00Z": 0,\n "2026-01-12T20:38:00Z": -1,\n "2026-01-12T20:43:00Z": -1,\n "2026-01-12T20:47:00Z": 0,\n "2026-01-12T20:50:00Z": 1,\n "2026-01-12T20:57:00Z": 2,\n "2026-01-12T21:03:00Z": 4,\n "2026-01-12T21:08:00Z": 5,\n "2026-01-12T21:12:00Z": 6,\n "2026-01-12T21:15:00Z": 7,\n "2026-01-12T21:22:00Z": 8,\n "2026-01-12T21:28:00Z": 8,\n "2026-01-12T21:33:00Z": 8,\n "2026-01-12T21:37:00Z": 7,\n "2026-01-12T21:40:00Z": null,\n "2026-01-12T21:47:00Z": null,\n "2026-01-12T21:53:00Z": null,\n "2026-01-12T21:58:00Z": null,\n "2026-01-12T22:02:00Z": null,\n "2026-01-12T22:05:00Z": null,\n "2026-01-12T22:12:00Z": null,\n "2026-01-12T22:18:00Z": 2,\n "2026-01-12T22:23:00Z": 3,\n "2026-01-12T22:27:00Z": 5,\n "2026-01-12T22:30:00Z": 6,\n "2026-01-12T22:37:00Z": 8,\n "2026-01-12T22:43:00Z": 9,\n "2026-01-12T22:48:00Z": 10,\n "2026-01-12T22:52:00Z": 11,\n "2026-01-12T22:55:00Z": null,\n "2026-01-12T23:02:00Z": null,\n "2026-01-12T23:08:00Z": null,\n "2026-01-12T23:13:00Z": null,\n "2026-01-12T23:17:00Z": null,\n "2026-01-12T23:20:00Z": null,\n "2026-01-12T23:27:00Z": null,\n "2026-01-12T23:33:00Z": null,\n "2026-01-12T23:38:00Z": null,\n "2026-01-12T23:42:00Z": null,\n "2026-01-12T23:45:00Z": 6,\n "2026-01-12T23:52:00Z": 7,\n "2026-01-12T23:58:00Z": 8\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-01-12T00:00:00Z": 10,\n "2026-01-12T00:07:00Z": 12,\n "2026-01-12T00:13:00Z": 13,\n "2026-01-12T00:18:00Z": 14,\n "2026-01-12T00:22:00Z": 15,\n "2026-01-12T00:25:00Z": 15,\n "2026-01-12T00:32:00Z": 15,\n "2026-01-12T00:38:00Z": 14,\n "2026-01-12T00:43:00Z": 13,\n "2026-01-12T00:47:00Z": 12,\n "2026-01-12T00:50:00Z": 11,\n "2026-01-12T00:57:00Z": 10,\n "2026-01-12T01:03:00Z": null,\n "2026-01-12T01:08:00Z": 9,\n "2026-01-12T01:12:00Z": 9,\n "2026-01-12T01:15:00Z": 10,\n "2026-01-12T01:22:00Z": 11,\n "2026-01-12T01:28:00Z": 12,\n "2026-01-12T01:33:00Z": 14,\n "2026-01-12T01:37:00Z": 15,\n "2026-01-12T01:40:00Z": 17,\n "2026-01-12T01:47:00Z": 18,\n "2026-01-12T01:53:00Z": 18,\n "2026-01-12T01:58:00Z": 19,\n "2026-01-12T02:02:00Z": 18,\n "2026-01-12T02:05:00Z": 18,\n "2026-01-12T02:12:00Z": 16,\n "2026-01-12T02:18:00Z": 15,\n "2026-01-12T02:23:00Z": 14,\n "2026-01-12T02:27:00Z": 13,\n "2026-01-12T02:30:00Z": null,\n "2026-01-12T02:37:00Z": null,\n "2026-01-12T02:43:00Z": null,\n "2026-01-12T02:48:00Z": 13,\n "2026-01-12T02:52:00Z": 14,\n "2026-01-12T02:55:00Z": 15,\n "2026-01-12T03:02:00Z": 16,\n "2026-01-12T03:08:00Z": 18,\n "2026-01-12T03:13:00Z": 19,\n "2026-01-12T03:17:00Z": 20,\n "2026-01-12T03:20:00Z": 21,\n "2026-01-12T03:27:00Z": 21,\n "2026-01-12T03:33:00Z": 20,\n "2026-01-12T03:38:00Z": 20,\n "2026-01-12T03:42:00Z": 19,\n "2026-01-12T03:45:00Z": 17,\n "2026-01-12T03:52:00Z": 16,\n "2026-01-12T03:58:00Z": 15,\n "2026-01-12T04:03:00Z": 14,\n "2026-01-12T04:07:00Z": 13,\n "2026-01-12T04:10:00Z": 13,\n "2026-01-12T04:17:00Z": 14,\n "2026-01-12T04:23:00Z": 15,\n "2026-01-12T04:28:00Z": 16,\n "2026-01-12T04:32:00Z": null,\n "2026-01-12T04:35:00Z": null,\n "2026-01-12T04:42:00Z": null,\n "2026-01-12T04:48:00Z": null,\n "2026-01-12T04:53:00Z": null,\n "2026-01-12T04:57:00Z": null,\n "2026-01-12T05:00:00Z": 21,\n "2026-01-12T05:07:00Z": 20,\n "2026-01-12T05:13:00Z": 19,\n "2026-01-12T05:18:00Z": 17,\n "2026-01-12T05:22:00Z": 16,\n "2026-01-12T05:25:00Z": 15,\n "2026-01-12T05:32:00Z": 14,\n "2026-01-12T05:38:00Z": 13,\n "2026-01-12T05:43:00Z": 13,\n "2026-01-12T05:47:00Z": 14,\n "2026-01-12T05:50:00Z": 15,\n "2026-01-12T05:57:00Z": 16,\n "2026-01-12T06:03:00Z": 17,\n "2026-01-12T06:08:00Z": 18,\n "2026-01-12T06:12:00Z": 19,\n "2026-01-12T06:15:00Z": 20,\n "2026-01-12T06:22:00Z": 21,\n "2026-01-12T06:28:00Z": 21,\n "2026-01-12T06:33:00Z": 20,\n "2026-01-12T06:37:00Z": 19,\n "2026-01-12T06:40:00Z": 18,\n "2026-01-12T06:47:00Z": 16,\n "2026-01-12T06:53:00Z": 15,\n "2026-01-12T06:58:00Z": 14,\n "2026-01-12T07:02:00Z": 13,\n "2026-01-12T07:05:00Z": 12,\n "2026-01-12T07:12:00Z": 12,\n "2026-01-12T07:18:00Z": 12,\n "2026-01-12T07:23:00Z": 13,\n "2026-01-12T07:27:00Z": 14,\n "2026-01-12T07:30:00Z": 15,\n "2026-01-12T07:37:00Z": 17,\n "2026-01-12T07:43:00Z": 18,\n "2026-01-12T07:48:00Z": 19,\n "2026-01-12T07:52:00Z": 19,\n "2026-01-12T07:55:00Z": 19,\n "2026-01-12T08:02:00Z": null,\n "2026-01-12T08:08:00Z": null,\n "2026-01-12T08:13:00Z": null,\n "2026-01-12T08:17:00Z": null,\n "2026-01-12T08:20:00Z": null,\n "2026-01-12T08:27:00Z": null,\n "2026-01-12T08:33:00Z": null,\n "2026-01-12T08:38:00Z": null,\n "2026-01-12T08:42:00Z": null,\n "2026-01-12T08:45:00Z": null,\n "2026-01-12T08:52:00Z": null,\n "2026-01-12T08:58:00Z": null,\n "2026-01-12T09:03:00Z": null,\n "2026-01-12T09:07:00Z": null,\n "2026-01-12T09:10:00Z": null,\n "2026-01-12T09:17:00Z": null,\n "2026-01-12T09:23:00Z": null,\n "2026-01-12T09:28:00Z": null,\n "2026-01-12T09:32:00Z": null,\n "2026-01-12T09:35:00Z": null,\n "2026-01-12T09:42:00Z": null,\n "2026-01-12T09:48:00Z": null,\n "2026-01-12T09:53:00Z": null,\n "2026-01-12T09:57:00Z": null,\n "2026-01-12T10:00:00Z": null,\n "2026-01-12T10:07:00Z": null,\n "2026-01-12T10:13:00Z": null,\n "2026-01-12T10:18:00Z": null,\n "2026-01-12T10:22:00Z": null,\n "2026-01-12T10:25:00Z": null,\n "2026-01-12T10:32:00Z": 12,\n "2026-01-12T10:38:00Z": 13,\n "2026-01-12T10:43:00Z": 14,\n "2026-01-12T10:47:00Z": 15,\n "2026-01-12T10:50:00Z": 15,\n "2026-01-12T10:57:00Z": 15,\n "2026-01-12T11:03:00Z": 15,\n "2026-01-12T11:08:00Z": 14,\n "2026-01-12T11:12:00Z": 12,\n "2026-01-12T11:15:00Z": 11,\n "2026-01-12T11:22:00Z": 9,\n "2026-01-12T11:28:00Z": 8,\n "2026-01-12T11:33:00Z": 7,\n "2026-01-12T11:37:00Z": 6,\n "2026-01-12T11:40:00Z": null,\n "2026-01-12T11:47:00Z": null,\n "2026-01-12T11:53:00Z": 8,\n "2026-01-12T11:58:00Z": 9,\n "2026-01-12T12:02:00Z": 10,\n "2026-01-12T12:05:00Z": 11,\n "2026-01-12T12:12:00Z": 12,\n "2026-01-12T12:18:00Z": 13,\n "2026-01-12T12:23:00Z": 14,\n "2026-01-12T12:27:00Z": 14,\n "2026-01-12T12:30:00Z": 13,\n "2026-01-12T12:37:00Z": 12,\n "2026-01-12T12:43:00Z": 11,\n "2026-01-12T12:48:00Z": 9,\n "2026-01-12T12:52:00Z": 8,\n "2026-01-12T12:55:00Z": 6,\n "2026-01-12T13:02:00Z": 5,\n "2026-01-12T13:08:00Z": 5,\n "2026-01-12T13:13:00Z": 5,\n "2026-01-12T13:17:00Z": 5,\n "2026-01-12T13:20:00Z": 6,\n "2026-01-12T13:27:00Z": 7,\n "2026-01-12T13:33:00Z": 8,\n "2026-01-12T13:38:00Z": 10,\n "2026-01-12T13:42:00Z": 11,\n "2026-01-12T13:45:00Z": null,\n "2026-01-12T13:52:00Z": null,\n "2026-01-12T13:58:00Z": null,\n "2026-01-12T14:03:00Z": null,\n "2026-01-12T14:07:00Z": null,\n "2026-01-12T14:10:00Z": null,\n "2026-01-12T14:17:00Z": null,\n "2026-01-12T14:23:00Z": null,\n "2026-01-12T14:28:00Z": 5,\n "2026-01-12T14:32:00Z": 4,\n "2026-01-12T14:35:00Z": 3,\n "2026-01-12T14:42:00Z": 3,\n "2026-01-12T14:48:00Z": 3,\n "2026-01-12T14:53:00Z": 4,\n "2026-01-12T14:57:00Z": 5,\n "2026-01-12T15:00:00Z": 7,\n "2026-01-12T15:07:00Z": 8,\n "2026-01-12T15:13:00Z": 9,\n "2026-01-12T15:18:00Z": 10,\n "2026-01-12T15:22:00Z": 10,\n "2026-01-12T15:25:00Z": 10,\n "2026-01-12T15:32:00Z": 9,\n "2026-01-12T15:38:00Z": 8,\n "2026-01-12T15:43:00Z": 7,\n "2026-01-12T15:47:00Z": 6,\n "2026-01-12T15:50:00Z": 4,\n "2026-01-12T15:57:00Z": 3,\n "2026-01-12T16:03:00Z": 2,\n "2026-01-12T16:08:00Z": 1,\n "2026-01-12T16:12:00Z": 1,\n "2026-01-12T16:15:00Z": 1,\n "2026-01-12T16:22:00Z": 2,\n "2026-01-12T16:28:00Z": 3,\n "2026-01-12T16:33:00Z": 5,\n "2026-01-12T16:37:00Z": 6,\n "2026-01-12T16:40:00Z": null,\n "2026-01-12T16:47:00Z": null,\n "2026-01-12T16:53:00Z": null,\n "2026-01-12T16:58:00Z": null,\n "2026-01-12T17:02:00Z": 7,\n "2026-01-12T17:05:00Z": 6,\n "2026-01-12T17:12:00Z": 5,\n "2026-01-12T17:18:00Z": 4,\n "2026-01-12T17:23:00Z": 2,\n "2026-01-12T17:27:00Z": 1,\n "2026-01-12T17:30:00Z": 0,\n "2026-01-12T17:37:00Z": -1,\n "2026-01-12T17:43:00Z": -1,\n "2026-01-12T17:48:00Z": 0,\n "2026-01-12T17:52:00Z": 1,\n "2026-01-12T17:55:00Z": 2,\n "2026-01-12T18:02:00Z": 3,\n "2026-01-12T18:08:00Z": 4,\n "2026-01-12T18:13:00Z": 5,\n "2026-01-12T18:17:00Z": 6,\n "2026-01-12T18:20:00Z": null,\n "2026-01-12T18:27:00Z": null,\n "2026-01-12T18:33:00Z": null,\n "2026-01-12T18:38:00Z": null,\n "2026-01-12T18:42:00Z": null,\n "2026-01-12T18:45:00Z": null,\n "2026-01-12T18:52:00Z": null,\n "2026-01-12T18:58:00Z": null,\n "2026-01-12T19:03:00Z": null,\n "2026-01-12T19:07:00Z": -1,\n "2026-01-12T19:10:00Z": -1,\n "2026-01-12T19:17:00Z": -1,\n "2026-01-12T19:23:00Z": 0,\n "2026-01-12T19:28:00Z": 1,\n "2026-01-12T19:32:00Z": 2,\n "2026-01-12T19:35:00Z": 4,\n "2026-01-12T19:42:00Z": null,\n "2026-01-12T19:48:00Z": null,\n "2026-01-12T19:53:00Z": null,\n "2026-01-12T19:57:00Z": null,\n "2026-01-12T20:00:00Z": null,\n "2026-01-12T20:07:00Z": 5,\n "2026-01-12T20:13:00Z": 4,\n "2026-01-12T20:18:00Z": 3,\n "2026-01-12T20:22:00Z": 1,\n "2026-01-12T20:25:00Z": 0,\n "2026-01-12T20:32:00Z": 0,\n "2026-01-12T20:38:00Z": -1,\n "2026-01-12T20:43:00Z": -1,\n "2026-01-12T20:47:00Z": 0,\n "2026-01-12T20:50:00Z": 1,\n "2026-01-12T20:57:00Z": 2,\n "2026-01-12T21:03:00Z": 4,\n "2026-01-12T21:08:00Z": 5,\n "2026-01-12T21:12:00Z": 6,\n "2026-01-12T21:15:00Z": 7,\n "2026-01-12T21:22:00Z": 8,\n "2026-01-12T21:28:00Z": 8,\n "2026-01-12T21:33:00Z": 8,\n "2026-01-12T21:37:00Z": 7,\n "2026-01-12T21:40:00Z": null,\n "2026-01-12T21:47:00Z": null,\n "2026-01-12T21:53:00Z": null,\n "2026-01-12T21:58:00Z": null,\n "2026-01-12T22:02:00Z": null,\n "2026-01-12T22:05:00Z": null,\n "2026-01-12T22:12:00Z": null,\n "2026-01-12T22:18:00Z": 2,\n "2026-01-12T22:23:00Z": 3,\n "2026-01-12T22:27:00Z": 5,\n "2026-01-12T22:30:00Z": 6,\n "2026-01-12T22:37:00Z": 8,\n "2026-01-12T22:43:00Z": 9,\n "2026-01-12T22:48:00Z": 10,\n "2026-01-12T22:52:00Z": 11,\n "2026-01-12T22:55:00Z": null,\n "2026-01-12T23:02:00Z": null,\n "2026-01-12T23:08:00Z": null,\n "2026-01-12T23:13:00Z": null,\n "2026-01-12T23:17:00Z": null,\n "2026-01-12T23:20:00Z": null,\n "2026-01-12T23:27:00Z": null,\n "2026-01-12T23:33:00Z": null,\n "2026-01-12T23:38:00Z": null,\n "2026-01-12T23:42:00Z": null,\n "2026-01-12T23:45:00Z": 6,\n "2026-01-12T23:52:00Z": 7,\n "2026-01-12T23:58:00Z": 8\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/integrate_time_series_100_543ffb9a-07da-49b8-83f3-64202de91e54.py b/runtime/transformations/components/time-series-base-components/integrate_time_series_100_543ffb9a-07da-49b8-83f3-64202de91e54.py new file mode 100644 index 000000000..fa09ab624 --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/integrate_time_series_100_543ffb9a-07da-49b8-83f3-64202de91e54.py @@ -0,0 +1,609 @@ +"""Documentation for Integrate Time Series + +# Integrate Time Series + +## Description +Component to integrate a time series on a datetime axis. It supports irregular +sampling, optional periodic resets, and can return either the cumulative +integrated series or one sum per period. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric. +- **output_mode** (String, default value: "series_integrated"): + Selects which result series the component returns. + - `series_integrated`: returns the cumulative integrated series. + - `period_sums`: returns one sum per reset period. If `reset` is `null`, + one total sum for the whole series is returned and indexed by the first + timestamp of the input series. +- **method** (String, default value: "trapezoidal"): + Integration method. Must be one of `"trapezoidal"`, `"rectangle_left"`, or + `"rectangle_right"`. + - `trapezoidal`: assumes the value changes linearly between two timestamps. + This is a good default for smooth signals. + - `rectangle_left`: uses the left value for the whole interval until the + next timestamp. + - `rectangle_right`: uses the right value for the whole interval and applies + it backward to the interval. +- **gap_handling** (String, default value: "break"): + Controls how missing values are handled during integration. + - `break`: does not integrate across intervals that contain missing values. + This is the safer default. + - `ignore`: skips missing observations and integrates between the remaining + valid points. This is a more aggressive option because it connects valid + datapoints across gaps. +- **max_gap** (String, default value: null): + Optional maximum time gap allowed for integration across two neighboring + valid timestamps. If the time difference is larger than this value, the + interval is skipped. Use a fixed timedelta string such as `2h`, `30min`, + or `1D`, or `null` to disable this check. +- **reset** (String, default value: null): + Optional reset period for the cumulative output. Must be one of `hourly`, + `daily`, `weekly`, `monthly`, `yearly`, or `null`. +- **time_unit** (String, default value: "h"): + Time unit used to scale the time differences during integration. Must be one + of `s`, `min`, `h`, or `d`. + For example, if the values are in `kW` and `time_unit="h"`, the integrated + output is in `kWh`. + +## Outputs +- **result_series** (Pandas Series): + The selected result series. Depending on `output_mode`, this is either the + cumulative integrated series or the period sums. For + `output_mode="series_integrated"`, additional timestamps can appear at reset + boundaries if an integration interval crosses such a boundary. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. Depending on `gap_handling`, missing values are either treated as hard + breaks or skipped during integration. +3. If `max_gap` is set, intervals larger than this threshold are skipped even + if both endpoint values are valid. +4. The integral is calculated interval by interval on the datetime axis using + the selected `method`. +5. The `time_unit` input scales the time differences, for example to hours when + converting power to energy. +6. If `reset` is set, intervals that cross a period boundary are split so that + the cumulative result can be represented correctly across the boundary. This + can add extra timestamps at reset boundaries in + `output_mode="series_integrated"` if no original datapoint exists exactly at + that boundary. +7. If `output_mode="period_sums"`, one sum per reset period is returned. + Without a reset, one total sum for the whole series is returned and indexed + by the first timestamp of the input series. +8. If the series should first be put on a regular time grid or missing + timestamps should be handled explicitly, use `handle_gaps_and_missing_data` + upstream and keep `gap_handling="break"` here. + +## Example +```json +{ + "timeseries": { + "2026-03-01T20:00:00Z": 5.0, + "2026-03-01T21:00:00Z": 5.5, + "2026-03-01T22:00:00Z": 5.2, + "2026-03-01T23:00:00Z": null, + "2026-03-02T00:00:00Z": 6.0, + "2026-03-02T01:00:00Z": 5.8, + "2026-03-02T02:00:00Z": 6.1, + "2026-03-02T03:00:00Z": 5.9, + "2026-03-02T04:00:00Z": 6.1, + "2026-03-02T05:00:00Z": 5.8, + "2026-03-02T06:00:00Z": 5.6, + "2026-03-02T07:00:00Z": 5.5, + "2026-03-02T08:00:00Z": 5.7 + }, + "output_mode": "series_integrated", + "method": "trapezoidal", + "gap_handling": "break", + "max_gap": "2h", + "reset": "daily", + "time_unit": "h" +} +``` + +Expected output: +```json +{ + "result_series": { + "2026-03-01T20:00:00Z": 0.0, + "2026-03-01T21:00:00Z": 5.25, + "2026-03-01T22:00:00Z": 10.6, + "2026-03-01T23:00:00Z": 10.6, + "2026-03-02T00:00:00Z": 0.0, + "2026-03-02T01:00:00Z": 5.9, + "2026-03-02T02:00:00Z": 11.85, + "2026-03-02T03:00:00Z": 17.85, + "2026-03-02T04:00:00Z": 23.85, + "2026-03-02T05:00:00Z": 29.8, + "2026-03-02T06:00:00Z": 35.5, + "2026-03-02T07:00:00Z": 41.05, + "2026-03-02T08:00:00Z": 46.65 + } +} +``` + +Second example with period sums: +```json +{ + "timeseries": { + "2026-03-01T20:00:00Z": 5.0, + "2026-03-01T21:00:00Z": 5.5, + "2026-03-01T22:00:00Z": 5.2, + "2026-03-01T23:00:00Z": null, + "2026-03-02T00:00:00Z": 6.0, + "2026-03-02T01:00:00Z": 5.8, + "2026-03-02T02:00:00Z": 6.1, + "2026-03-02T03:00:00Z": 5.9, + "2026-03-02T04:00:00Z": 6.1, + "2026-03-02T05:00:00Z": 5.8, + "2026-03-02T06:00:00Z": 5.6, + "2026-03-02T07:00:00Z": 5.5, + "2026-03-02T08:00:00Z": 5.7 + }, + "output_mode": "period_sums", + "method": "trapezoidal", + "gap_handling": "break", + "max_gap": "2h", + "reset": "daily", + "time_unit": "h" +} +``` + +Expected output: +```json +{ + "result_series": { + "2026-03-01T00:00:00Z": 10.6, + "2026-03-02T00:00:00Z": 46.65 + } +} +``` + +Notes: +- `gap_handling="break"` is the recommended default when missing values should + stop the integration. +- `gap_handling="ignore"` should only be used intentionally, because it + connects valid datapoints across gaps. +- If `reset` is active and an interval crosses a reset boundary, the cumulative + output can contain an additional timestamp exactly at that boundary so that + the reset is represented correctly. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from hdutils import ComponentInputValidationException + +TIME_UNIT_IN_SECONDS = { + "s": 1.0, + "min": 60.0, + "h": 3600.0, + "d": 86400.0, +} + + +def parse_fixed_timedelta_string(value: str, input_name: str) -> pd.Timedelta: + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a valid fixed timedelta string like '30min', '2h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + try: + nanos = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '30min', '2h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + return pd.to_timedelta(nanos, unit="ns") + + +def validate_inputs( + timeseries: pd.Series, + output_mode: str, + method: str, + gap_handling: str, + max_gap: str | None, + reset: str | None, + time_unit: str, +) -> pd.Timedelta | None: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_datetime64_any_dtype(timeseries.index): + raise ComponentInputValidationException( + "timeseries index must be datetime", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not np.isfinite(timeseries.dropna().to_numpy(dtype=float)).all(): + raise ComponentInputValidationException( + "timeseries must not contain inf or -inf values", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.dropna().empty: + raise ComponentInputValidationException( + "timeseries must contain at least one non-missing numeric value", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if method not in {"trapezoidal", "rectangle_left", "rectangle_right"}: + raise ComponentInputValidationException( + "method must be one of 'trapezoidal', 'rectangle_left', 'rectangle_right'", + error_code="422", + invalid_component_inputs=["method"], + ) + if gap_handling not in {"ignore", "break"}: + raise ComponentInputValidationException( + "gap_handling must be one of 'break', 'ignore'", + error_code="422", + invalid_component_inputs=["gap_handling"], + ) + if max_gap is not None: + if not isinstance(max_gap, str): + raise ComponentInputValidationException( + "max_gap must be null or a fixed timedelta string like '2h' or '30min'", + error_code="422", + invalid_component_inputs=["max_gap"], + ) + normalized_max_gap = parse_fixed_timedelta_string(max_gap, "max_gap") + if normalized_max_gap <= pd.Timedelta(0): + raise ComponentInputValidationException( + "max_gap must be greater than 0", + error_code="422", + invalid_component_inputs=["max_gap"], + ) + else: + normalized_max_gap = None + if reset not in {None, "hourly", "daily", "weekly", "monthly", "yearly"}: + raise ComponentInputValidationException( + "reset must be one of null, 'hourly', 'daily', 'weekly', 'monthly', 'yearly'", + error_code="422", + invalid_component_inputs=["reset"], + ) + if output_mode not in {"series_integrated", "period_sums"}: + raise ComponentInputValidationException( + "output_mode must be one of 'series_integrated', 'period_sums'", + error_code="422", + invalid_component_inputs=["output_mode"], + ) + if time_unit not in TIME_UNIT_IN_SECONDS: + raise ComponentInputValidationException( + "time_unit must be one of 's', 'min', 'h', 'd'", + error_code="422", + invalid_component_inputs=["time_unit"], + ) + return normalized_max_gap + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared + + +def build_integration_intervals( + output_index: pd.DatetimeIndex, + series: pd.Series, + gap_handling: str, + max_gap: pd.Timedelta | None, +) -> list[tuple[pd.Timestamp, pd.Timestamp, float, float]]: + intervals: list[tuple[pd.Timestamp, pd.Timestamp, float, float]] = [] + if gap_handling == "ignore": + valid_series = series.dropna() + if len(valid_series) < 2: + return intervals + for i in range(1, len(valid_series)): + ts0 = valid_series.index[i - 1] + ts1 = valid_series.index[i] + if max_gap is not None and (ts1 - ts0) > max_gap: + continue + v0 = float(valid_series.iloc[i - 1]) + v1 = float(valid_series.iloc[i]) + intervals.append((ts0, ts1, v0, v1)) + return intervals + + for i in range(1, len(output_index)): + ts0 = output_index[i - 1] + ts1 = output_index[i] + if max_gap is not None and (ts1 - ts0) > max_gap: + continue + v0 = series.iloc[i - 1] + v1 = series.iloc[i] + if pd.isna(v0) or pd.isna(v1): + continue + intervals.append((ts0, ts1, float(v0), float(v1))) + return intervals + + +def determine_period_start(timestamp: pd.Timestamp, reset: str | None) -> pd.Timestamp: + if reset is None: + return timestamp + normalized = timestamp.normalize() + if reset == "hourly": + return timestamp.floor("h") + if reset == "daily": + return normalized + if reset == "weekly": + return normalized - pd.Timedelta(days=int(timestamp.weekday())) + if reset == "monthly": + return normalized.replace(day=1) + return normalized.replace(month=1, day=1) + + +def determine_next_period_start( + timestamp: pd.Timestamp, + reset: str | None, +) -> pd.Timestamp: + period_start = determine_period_start(timestamp, reset) + if reset == "hourly": + return period_start + pd.Timedelta(hours=1) + if reset == "daily": + return period_start + pd.Timedelta(days=1) + if reset == "weekly": + return period_start + pd.Timedelta(days=7) + if reset == "monthly": + return period_start + pd.offsets.MonthBegin(1) + if reset == "yearly": + return period_start + pd.offsets.YearBegin(1) + return timestamp + + +def split_interval_at_reset_boundaries( + start_timestamp: pd.Timestamp, + end_timestamp: pd.Timestamp, + reset: str | None, +) -> list[tuple[pd.Timestamp, pd.Timestamp]]: + if reset is None: + return [(start_timestamp, end_timestamp)] + + parts: list[tuple[pd.Timestamp, pd.Timestamp]] = [] + current_start = start_timestamp + while current_start < end_timestamp: + next_boundary = determine_next_period_start(current_start, reset) + current_end = min(end_timestamp, next_boundary) + parts.append((current_start, current_end)) + current_start = current_end + return parts + + +def calculate_segment_area( + interval_start: pd.Timestamp, + interval_end: pd.Timestamp, + start_value: float, + end_value: float, + segment_start: pd.Timestamp, + segment_end: pd.Timestamp, + method: str, + time_unit: str, +) -> float: + interval_seconds = (interval_end - interval_start).total_seconds() + if interval_seconds <= 0: + return 0.0 + + segment_seconds = (segment_end - segment_start).total_seconds() + if segment_seconds <= 0: + return 0.0 + + scale = TIME_UNIT_IN_SECONDS[time_unit] + segment_duration = segment_seconds / scale + + if method == "rectangle_left": + return start_value * segment_duration + if method == "rectangle_right": + return end_value * segment_duration + + start_fraction = (segment_start - interval_start).total_seconds() / interval_seconds + end_fraction = (segment_end - interval_start).total_seconds() / interval_seconds + value_at_segment_start = start_value + (end_value - start_value) * start_fraction + value_at_segment_end = start_value + (end_value - start_value) * end_fraction + return 0.5 * (value_at_segment_start + value_at_segment_end) * segment_duration + + +def initialize_period_sums( + output_index: pd.DatetimeIndex, + reset: str | None, +) -> dict[pd.Timestamp, float]: + if len(output_index) == 0: + return {} + if reset is None: + return {output_index[0]: 0.0} + + period_starts = [determine_period_start(ts, reset) for ts in output_index] + unique_period_starts = pd.Index(period_starts).unique().sort_values() + return {pd.Timestamp(ts): 0.0 for ts in unique_period_starts} + + +def calculate_integrated_series_and_period_sums( + series: pd.Series, + method: str, + gap_handling: str, + max_gap: pd.Timedelta | None, + reset: str | None, + time_unit: str, +) -> tuple[pd.Series, pd.Series]: + original_index = series.index + intervals = build_integration_intervals(original_index, series, gap_handling, max_gap) + + internal_boundary_timestamps: set[pd.Timestamp] = set() + for interval_start, interval_end, _, _ in intervals: + interval_parts = split_interval_at_reset_boundaries(interval_start, interval_end, reset) + for _, part_end in interval_parts[:-1]: + internal_boundary_timestamps.add(part_end) + + integrated_index = original_index.union(pd.DatetimeIndex(sorted(internal_boundary_timestamps))) + incremental_area_at_timestamp = dict.fromkeys(integrated_index, 0.0) + period_sums_by_period = initialize_period_sums(integrated_index, reset) + + for interval_start, interval_end, start_value, end_value in intervals: + interval_parts = split_interval_at_reset_boundaries(interval_start, interval_end, reset) + for part_start, part_end in interval_parts: + part_area = calculate_segment_area( + interval_start, + interval_end, + start_value, + end_value, + part_start, + part_end, + method, + time_unit, + ) + if reset is None: + period_key = integrated_index[0] + else: + period_key = determine_period_start(part_start, reset) + period_sums_by_period[period_key] = ( + period_sums_by_period.get(period_key, 0.0) + part_area + ) + incremental_area_at_timestamp[part_end] = ( + incremental_area_at_timestamp.get(part_end, 0.0) + part_area + ) + + integrated_values = pd.Series(0.0, index=integrated_index, name="series_integrated") + running_total = 0.0 + current_period_start: pd.Timestamp | None = None + for timestamp in integrated_index: + if timestamp in internal_boundary_timestamps: + running_total += incremental_area_at_timestamp.get(timestamp, 0.0) + integrated_values.loc[timestamp] = running_total + running_total = 0.0 + current_period_start = determine_period_start(timestamp, reset) + continue + + if reset is not None: + timestamp_period_start = determine_period_start(timestamp, reset) + if current_period_start is None or timestamp_period_start != current_period_start: + running_total = 0.0 + current_period_start = timestamp_period_start + running_total += incremental_area_at_timestamp.get(timestamp, 0.0) + integrated_values.loc[timestamp] = running_total + + period_sums = pd.Series(period_sums_by_period, dtype=float, name="period_sums").sort_index() + + return integrated_values, period_sums + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "output_mode": {"data_type": "STRING", "default_value": "series_integrated"}, + "method": {"data_type": "STRING", "default_value": "trapezoidal"}, + "gap_handling": {"data_type": "STRING", "default_value": "break"}, + "max_gap": {"data_type": "STRING", "default_value": None}, + "reset": {"data_type": "STRING", "default_value": None}, + "time_unit": {"data_type": "STRING", "default_value": "h"}, + }, + "outputs": { + "result_series": {"data_type": "SERIES"}, + }, + "name": "Integrate Time Series", + "category": "Time Series Base Components", + "description": "Integrate a time series on a datetime axis.", + "version_tag": "1.0.0", + "id": "543ffb9a-07da-49b8-83f3-64202de91e54", + "revision_group_id": "3f56cedc-4241-4ae3-be5c-b85c1987fded", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + output_mode="series_integrated", + method="trapezoidal", + gap_handling="break", + max_gap=None, + reset=None, + time_unit="h", +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate the user inputs. + normalized_max_gap = validate_inputs( + timeseries, + output_mode, + method, + gap_handling, + max_gap, + reset, + time_unit, + ) + + # Step 2: Sort the input series and merge duplicate timestamps. + prepared = prepare_series(timeseries) + + # Step 3: Calculate both possible result series. + series_integrated, period_sums = calculate_integrated_series_and_period_sums( + prepared, + method, + gap_handling, + normalized_max_gap, + reset, + time_unit, + ) + + # Step 4: Select the configured result series. + if output_mode == "series_integrated": + result_series = series_integrated.rename("result_series") + else: + result_series = period_sums.rename("result_series") + + # Step 5: Return the selected output series. + return { + "result_series": result_series, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T20:00:00Z": 5.0,\n "2026-03-01T21:00:00Z": 5.5,\n "2026-03-01T22:00:00Z": 5.2,\n "2026-03-01T23:00:00Z": null,\n "2026-03-02T00:00:00Z": 6.0,\n "2026-03-02T01:00:00Z": 5.8,\n "2026-03-02T02:00:00Z": 6.1,\n "2026-03-02T03:00:00Z": 5.9,\n "2026-03-02T04:00:00Z": 6.1,\n "2026-03-02T05:00:00Z": 5.8,\n "2026-03-02T06:00:00Z": 5.6,\n "2026-03-02T07:00:00Z": 5.5,\n "2026-03-02T08:00:00Z": 5.7\n}' + }, + } + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T20:00:00Z": 5.0,\n "2026-03-01T21:00:00Z": 5.5,\n "2026-03-01T22:00:00Z": 5.2,\n "2026-03-01T23:00:00Z": null,\n "2026-03-02T00:00:00Z": 6.0,\n "2026-03-02T01:00:00Z": 5.8,\n "2026-03-02T02:00:00Z": 6.1,\n "2026-03-02T03:00:00Z": 5.9,\n "2026-03-02T04:00:00Z": 6.1,\n "2026-03-02T05:00:00Z": 5.8,\n "2026-03-02T06:00:00Z": 5.6,\n "2026-03-02T07:00:00Z": 5.5,\n "2026-03-02T08:00:00Z": 5.7\n}' + }, + } + ] +} diff --git a/runtime/transformations/components/time-series-base-components/resample_time_series_100_8d9180a9-9b95-4390-aeff-8a36f25f5d7b.py b/runtime/transformations/components/time-series-base-components/resample_time_series_100_8d9180a9-9b95-4390-aeff-8a36f25f5d7b.py new file mode 100644 index 000000000..fba15575d --- /dev/null +++ b/runtime/transformations/components/time-series-base-components/resample_time_series_100_8d9180a9-9b95-4390-aeff-8a36f25f5d7b.py @@ -0,0 +1,298 @@ +"""Documentation for Resample Time Series + +# Resample Time Series + +## Description +Component to bring a time series onto a target time grid. It supports regular +aggregation for downsampling and `asfreq` for pure reindexing without filling +missing values. + +## Inputs +- **timeseries** (Pandas Series): + The input time series. The index must contain timestamps, and the values + must be numeric. +- **target_frequency** (String): + Target time frequency such as `5min`, `15min`, `1h`, or `1D`. + Only fixed timedelta-like frequencies are supported. +- **aggregation_method** (String, default value: "mean"): + Resampling method. Must be one of `mean`, `median`, `min`, `max`, `sum`, + or `asfreq`. + - `mean`, `median`, `min`, `max`, `sum`: aggregate values inside each + target interval. + - `asfreq`: only place the series on the target grid. No aggregation and + no filling is performed. New timestamps stay `NaN` if no original value + exists there. +- **label_position** (String, default value: "left"): + Determines whether interval-based results are labeled with the left or the + right boundary of the resampling interval. Must be one of `left` or + `right`. For `aggregation_method="asfreq"`, this setting is normally not + the deciding factor. +- **closed** (String, default value: "left"): + Determines which side of each interval is closed for interval-based + resampling. Must be one of `left` or `right`. For + `aggregation_method="asfreq"`, this setting is normally not the deciding + factor. + +## Outputs +- **resampled_timeseries** (Pandas Series): + The resampled time series. + +## Details +1. The input series is sorted by time and duplicate timestamps are merged by mean. +2. The target frequency is validated as a fixed timedelta-like frequency. +3. For `aggregation_method="asfreq"`, the series is only aligned to the new + grid and no values are filled. +4. For all other aggregation methods, pandas resampling is used with the chosen + interval labeling and interval closure. +5. The component returns only the resampled series. +6. If missing values created by resampling should be filled afterwards, use + `handle_gaps_and_missing_data` as the next component. + +## Example +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 1.0, + "2026-03-01T00:05:00Z": 2.0, + "2026-03-01T00:10:00Z": 3.0, + "2026-03-01T00:15:00Z": 4.0, + "2026-03-01T00:20:00Z": 5.0, + "2026-03-01T00:25:00Z": 6.0, + "2026-03-01T00:30:00Z": 7.0 + }, + "target_frequency": "15min", + "aggregation_method": "mean", + "label_position": "left", + "closed": "left" +} +``` + +Expected output: +```json +{ + "resampled_timeseries": { + "2026-03-01T00:00:00Z": 2.0, + "2026-03-01T00:15:00Z": 5.0, + "2026-03-01T00:30:00Z": 7.0 + } +} +``` + +Second example with `asfreq`: +```json +{ + "timeseries": { + "2026-03-01T00:00:00Z": 10.0, + "2026-03-01T00:10:00Z": 12.0, + "2026-03-01T00:20:00Z": 14.0 + }, + "target_frequency": "5min", + "aggregation_method": "asfreq", + "label_position": "left", + "closed": "left" +} +``` + +Expected output: +```json +{ + "resampled_timeseries": { + "2026-03-01T00:00:00Z": 10.0, + "2026-03-01T00:05:00Z": null, + "2026-03-01T00:10:00Z": 12.0, + "2026-03-01T00:15:00Z": null, + "2026-03-01T00:20:00Z": 14.0 + } +} +``` +""" + +from __future__ import annotations + +import pandas as pd + +from hdutils import ComponentInputValidationException + +AGGREGATION_FUNCTIONS = { + "mean": "mean", + "median": "median", + "min": "min", + "max": "max", + "sum": "sum", + "asfreq": "asfreq", +} + + +def parse_fixed_frequency(value: str, input_name: str) -> str: + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a valid fixed timedelta string like '5min', '1h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + try: + _ = offset.nanos + except ValueError as exc: + raise ComponentInputValidationException( + f"{input_name} must be a fixed timedelta string like '5min', '1h', or '1D'", + error_code="422", + invalid_component_inputs=[input_name], + ) from exc + + return value + + +def validate_inputs( + timeseries: pd.Series, + target_frequency: str, + aggregation_method: str, + label_position: str, + closed: str, +) -> str: + if not isinstance(timeseries, pd.Series): + raise ComponentInputValidationException( + "timeseries must be a pandas Series", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if timeseries.empty: + raise ComponentInputValidationException( + "timeseries must not be empty", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not isinstance(timeseries.index, pd.DatetimeIndex): + raise ComponentInputValidationException( + "timeseries index must be a pandas DatetimeIndex", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + if not pd.api.types.is_numeric_dtype(timeseries): + raise ComponentInputValidationException( + "timeseries values must be numeric", + error_code="422", + invalid_component_inputs=["timeseries"], + ) + + if aggregation_method not in AGGREGATION_FUNCTIONS: + raise ComponentInputValidationException( + f"aggregation_method must be one of {sorted(AGGREGATION_FUNCTIONS)}", + error_code="422", + invalid_component_inputs=["aggregation_method"], + ) + + if label_position not in {"left", "right"}: + raise ComponentInputValidationException( + "label_position must be one of 'left' or 'right'", + error_code="422", + invalid_component_inputs=["label_position"], + ) + + if closed not in {"left", "right"}: + raise ComponentInputValidationException( + "closed must be one of 'left' or 'right'", + error_code="422", + invalid_component_inputs=["closed"], + ) + + return parse_fixed_frequency(target_frequency, "target_frequency") + + +def prepare_series(timeseries: pd.Series) -> pd.Series: + prepared = timeseries.sort_index() + if not prepared.index.is_unique: + prepared = prepared.groupby(level=0).mean() + return prepared + + +# ***** DO NOT EDIT LINES BELOW ***** +# These lines may be overwritten if component details or inputs/outputs change. +COMPONENT_INFO = { + "inputs": { + "timeseries": {"data_type": "SERIES"}, + "target_frequency": {"data_type": "STRING"}, + "aggregation_method": {"data_type": "STRING", "default_value": "mean"}, + "label_position": {"data_type": "STRING", "default_value": "left"}, + "closed": {"data_type": "STRING", "default_value": "left"}, + }, + "outputs": { + "resampled_timeseries": {"data_type": "SERIES"}, + }, + "name": "Resample Time Series", + "category": "Time Series Base Components", + "description": "Bring a time series onto a target time grid.", + "version_tag": "1.0.0", + "id": "8d9180a9-9b95-4390-aeff-8a36f25f5d7b", + "revision_group_id": "b4aa144e-cdfe-4ff5-8337-a5b8e35c1f15", + "state": "RELEASED", + "released_timestamp": "2026-05-11T06:00:00+00:00", +} + +from hdutils import parse_default_value # noqa: E402, F401 + + +def main( + *, + timeseries, + target_frequency, + aggregation_method="mean", + label_position="left", + closed="left", +): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + # Step 1: Validate inputs and the target frequency. + target_frequency = validate_inputs( + timeseries, + target_frequency, + aggregation_method, + label_position, + closed, + ) + + # Step 2: Sort the series and merge duplicate timestamps by mean. + prepared = prepare_series(timeseries) + + # Step 3: Resample either by pure reindexing or by interval aggregation. + resampler = prepared.resample( + target_frequency, + label=label_position, + closed=closed, + ) + if aggregation_method == "asfreq": + result = resampler.asfreq() + else: + result = resampler.agg(AGGREGATION_FUNCTIONS[aggregation_method]) + + # Step 4: Return the resampled series. + return { + "resampled_timeseries": result, + } + + +TEST_WIRING_FROM_PY_FILE_IMPORT = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 1.0,\n "2026-03-01T00:05:00Z": 2.0,\n "2026-03-01T00:10:00Z": 3.0,\n "2026-03-01T00:15:00Z": 4.0,\n "2026-03-01T00:20:00Z": 5.0,\n "2026-03-01T00:25:00Z": 6.0,\n "2026-03-01T00:30:00Z": 7.0\n}' + }, + }, + {"workflow_input_name": "target_frequency", "filters": {"value": "15min"}}, + ] +} + +RELEASE_WIRING = { + "input_wirings": [ + { + "workflow_input_name": "timeseries", + "filters": { + "value": '{\n "2026-03-01T00:00:00Z": 1.0,\n "2026-03-01T00:05:00Z": 2.0,\n "2026-03-01T00:10:00Z": 3.0,\n "2026-03-01T00:15:00Z": 4.0,\n "2026-03-01T00:20:00Z": 5.0,\n "2026-03-01T00:25:00Z": 6.0,\n "2026-03-01T00:30:00Z": 7.0\n}' + }, + }, + {"workflow_input_name": "target_frequency", "filters": {"value": "15min"}}, + ] +}