From 57c6c9478aa040a118597184451077d63cbed719 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 15 Jun 2026 14:40:10 +0200 Subject: [PATCH 01/21] draft solution for fork pr handling --- .github/workflows/acceptance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 96819e29..218c38b9 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,6 +37,10 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write + # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore + # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested + # by the reviewer(s) / maintainer(s) before merging. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -63,6 +67,8 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write + # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 7c990003ad82bf86428ec12eb7beb307d24074ec Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:57:41 +0200 Subject: [PATCH 02/21] Added functionality for PointsInTimeSeries to support string as well --- .../model/series/points_in_time_series.py | 88 +++++++++++++- .../series/points_in_time_series_test.py | 111 ++++++++++++++++++ 2 files changed, 193 insertions(+), 6 deletions(-) diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index ba46f831..192a73db 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Sized +import functools +from collections.abc import Callable, Sized import numpy as np import numpy.typing as npt @@ -15,6 +16,34 @@ FloatOrNaN = float | np.float64 +def _numeric_only(method: Callable) -> Callable: + """Decorator that rejects the wrapped method on a string-valued series. + + String-valued :class:`PointsInTimeSeries` support only sampling and equality + (``==`` / ``!=``); arithmetic, ordering and numeric reductions have no meaning + for them. Numpy would either raise (``-``, ``/``, ``mean``) or — worse — + silently succeed with a nonsensical result (``+`` concatenates, ``*`` repeats, + ``sum`` concatenates), so guard those methods explicitly and fail loudly. + + Applied to arithmetic, ordering-comparison and reduction methods. + + Raises + ------ + TypeError + When the decorated method is called on a string-valued series. + """ + + @functools.wraps(method) + def wrapper(self: PointsInTimeSeries, *args, **kwargs): + if self._is_string: + raise TypeError( + f"{method.__name__} is not supported for string-valued PointsInTimeSeries" + ) + return method(self, *args, **kwargs) + + return wrapper + + class PointsInTimeSeries: def __init__(self, tstarts: Sized, values: Sized): """ @@ -32,31 +61,62 @@ def __init__(self, tstarts: Sized, values: Sized): Array-like of values, one per time point. """ assert len(tstarts) == len(values) + # Timestamps are always numeric. Values may be numeric or string: + # string-valued series support sampling (``synchronized`` / ``.where``) + # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering + # and numeric reductions are rejected (see the ``@_numeric_only`` methods). + # An empty series has no observed value type, so it defaults to numeric + # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) - self.values = np.array(values, dtype=np.float64) + self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") + if self._is_string: + self.values = np.asarray(values, dtype=object) + else: + self.values = np.array(values, dtype=np.float64) def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. + For numeric values the element is a homogeneous ``[tstart, value]`` double + pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use + that homogeneous nested array, so their element is a ``(tstart, value)`` + struct with a double timestamp and a string value. + Returns ------- pyspark.sql.types.ArrayType - Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. - """ + Spark ArrayType matching ``get_data``'s shape for this series' value type. + """ + if self._is_string: + return T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) return T.ArrayType(T.ArrayType(T.DoubleType())) def get_data(self) -> list: """ - Returns the series as a list of [tstart, value] lists. + Returns the series as a list of ``[tstart, value]`` pairs. + + For numeric values this is a list of two-element double lists. For string + values, ``column_stack`` would coerce the timestamps to strings, so the + pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the + struct element type declared by :meth:`dtype`. Returns ------- list - List of [tstart, value] pairs. + List of ``[tstart, value]`` pairs. """ if len(self) == 0: return [] + if self._is_string: + return [[float(t), str(v)] for t, v in zip(self.tstarts, self.values, strict=True)] return np.column_stack([self.tstarts, self.values]).tolist() def __len__(self) -> int: @@ -354,34 +414,42 @@ def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTime return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values)) return PointsInTimeSeries(self.tstarts, operation(other, self.values)) + @_numeric_only def __add__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add another series or scalar to this series.""" return self._apply_basic_op(np.add, other) + @_numeric_only def __radd__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add this series to another series or scalar (reversed operands).""" return self._apply_basic_rop(np.add, other) + @_numeric_only def __sub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract another series or scalar from this series.""" return self._apply_basic_op(np.subtract, other) + @_numeric_only def __rsub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract this series from another series or scalar (reversed operands).""" return self._apply_basic_rop(np.subtract, other) + @_numeric_only def __mul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply this series by another series or scalar.""" return self._apply_basic_op(np.multiply, other) + @_numeric_only def __rmul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.multiply, other) + @_numeric_only def __truediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide this series by another series or scalar.""" return self._apply_basic_op(np.true_divide, other) + @_numeric_only def __rtruediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.true_divide, other) @@ -411,18 +479,22 @@ def __apply_op( idx = operation(self.values, other) return PointsInTime(self.tstarts[idx]) + @_numeric_only def __gt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than another.""" return self.__apply_op(np.greater, other) + @_numeric_only def __ge__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than or equal to another.""" return self.__apply_op(np.greater_equal, other) + @_numeric_only def __lt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than another.""" return self.__apply_op(np.less, other) + @_numeric_only def __le__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than or equal to another.""" return self.__apply_op(np.less_equal, other) @@ -448,6 +520,7 @@ def count(self) -> int: """ return len(self) + @_numeric_only def sum(self) -> FloatOrNaN: """ Returns the sum of the values. @@ -461,6 +534,7 @@ def sum(self) -> FloatOrNaN: return np.nan return np.sum(self.values) + @_numeric_only def mean(self) -> FloatOrNaN: """ Returns the mean of the values. @@ -474,6 +548,7 @@ def mean(self) -> FloatOrNaN: return np.nan return np.mean(self.values) + @_numeric_only def min(self) -> FloatOrNaN: """ Returns the minimum value. @@ -487,6 +562,7 @@ def min(self) -> FloatOrNaN: return np.nan return np.min(self.values) + @_numeric_only def max(self) -> FloatOrNaN: """ Returns the maximum value. diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index b957d965..77f8662b 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -152,6 +152,117 @@ def test_aggregations_empty(): assert pts.count() == 0 +# --- string values ------------------------------------------------------------------------------ +# String-valued series support sampling and equality only; arithmetic, ordering +# and numeric reductions are rejected. Timestamps stay numeric regardless. + + +def test_string_values_stored_as_object_with_numeric_timestamps(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + assert pts._is_string is True + assert pts.values.dtype == object + assert pts.tstarts.dtype == np.float64 + nptest.assert_array_equal(pts.values, ["P108B", "U0046", "P108B"]) + + +def test_empty_series_defaults_to_numeric(): + # No observed value type -> numeric (backward-compatible default). + assert PointsInTimeSeries.empty()._is_string is False + + +def test_numeric_series_is_not_string(): + assert PointsInTimeSeries([0, 1], [10, 20])._is_string is False + + +def test_string_eq_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + result = pts == "P108B" + assert isinstance(result, PointsInTime) + nptest.assert_array_equal(result.tstarts, [1, 3]) + + +def test_string_ne_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + nptest.assert_array_equal((pts != "P108B").tstarts, [2]) + + +def test_string_eq_series_matches_on_value_and_timestamp(): + p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]) + p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"]) + # Common timestamps {2,3}; values equal only at t=3 ("C" == "C"). + nptest.assert_array_equal((p1 == p2).tstarts, [3]) + + +def test_string_synchronized_with_sample_series_samples_values(): + pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"]) + s = SampleSeries([0, 10, 20], [10, 20, 30], [1, 2, 3]) + a, b = pts.synchronized(s) + nptest.assert_array_equal(a.tstarts, [5, 15, 25]) + nptest.assert_array_equal(a.values, ["a", "b", "c"]) + nptest.assert_array_equal(b.values, [1, 2, 3]) + + +def test_string_get_data_pairs_double_timestamp_with_string_value(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.get_data() == [[1.0, "P108B"], [2.0, "U0046"]] + + +def test_string_dtype_is_struct_of_double_and_string(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.dtype() == T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p + "x", + lambda p: "x" + p, + lambda p: p - 1, + lambda p: 1 - p, + lambda p: p * 2, + lambda p: p / 2, + ], +) +def test_string_arithmetic_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p > "A", + lambda p: p >= "A", + lambda p: p < "Z", + lambda p: p <= "Z", + ], +) +def test_string_ordering_comparison_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize("reduction", ["sum", "mean", "min", "max"]) +def test_string_reductions_raise(reduction): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + getattr(pts, reduction)() + + +def test_string_count_is_allowed(): + # count is structural (not value-dependent), so it works for strings. + assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]).count() == 3 + + # --- plane_sweep -------------------------------------------------------------------------------- From 26cf0e6f641818d6cbc0bdd21c238e2bf42b75b9 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:59:22 +0200 Subject: [PATCH 03/21] wip corrected github action --- .github/workflows/acceptance.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 218c38b9..96819e29 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,10 +37,6 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write - # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore - # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested - # by the reviewer(s) / maintainer(s) before merging. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -67,8 +63,6 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write - # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 9119fa36d43401b96177c9cdf70910bedb6a7163 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 07:09:40 +0200 Subject: [PATCH 04/21] added update-api-docs to Makefile ran update-api-docs --- Makefile | 5 ++++- .../model/series/points_in_time_series.md | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3d13e279..445f0aba 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,9 @@ coverage: build: uv build --require-hashes --build-constraints=.build-constraints.txt +update-api-docs: + cd docs/impulse && uv run pydoc-markdown + lock-dependencies: UV_LOCKED := 0 lock-dependencies: uv lock @@ -56,4 +59,4 @@ fork-sync: ./.github/scripts/fork-sync-pr.sh $(PR) .DEFAULT: all -.PHONY: all clean dev lint fmt test coverage build lock-dependencies fork-sync +.PHONY: all clean dev lint fmt test coverage build update-api-docs lock-dependencies fork-sync diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index 50cbf77b..ef8b12ce 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -37,9 +37,14 @@ def dtype() Returns the Spark data type for PointsInTimeSeries. +For numeric values the element is a homogeneous ``[tstart, value]`` double +pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use +that homogeneous nested array, so their element is a ``(tstart, value)`` +struct with a double timestamp and a string value. + **Returns**: -`pyspark.sql.types.ArrayType`: Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. +`pyspark.sql.types.ArrayType`: Spark ArrayType matching ``get_data``'s shape for this series' value type. #### get\_data @@ -47,11 +52,16 @@ Returns the Spark data type for PointsInTimeSeries. def get_data() -> list ``` -Returns the series as a list of [tstart, value] lists. +Returns the series as a list of ``[tstart, value]`` pairs. + +For numeric values this is a list of two-element double lists. For string +values, ``column_stack`` would coerce the timestamps to strings, so the +pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the +struct element type declared by :meth:`dtype`. **Returns**: -`list`: List of [tstart, value] pairs. +`list`: List of ``[tstart, value]`` pairs. #### \_\_len\_\_ From a189d949207c7b96bbc5067fee9896e01f3d9f65 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 14:18:30 +0200 Subject: [PATCH 05/21] added poi_series_integration.md and marked differences from design to impl --- demos/data/reporting/channel_metrics.csv | 32 +- demos/data/reporting/channel_tags.csv | 36 + demos/data/reporting/poi_channels.csv | 13 + demos/reporting_pipeline.ipynb | 95 +- .../metadata/time_series_expression.md | 77 +- .../analyze/query/query_builder.md | 34 + .../analyze/query/solvers/solver_config.md | 36 + .../model/series/points_in_time_series.md | 27 +- poi_series_integration.md | 896 ++++++++++++++++++ .../metadata/time_series_expression.py | 142 ++- .../analyze/query/query_builder.py | 44 + .../analyze/query/solvers/blob_solver.py | 7 +- .../analyze/query/solvers/default_solver.py | 127 ++- .../analyze/query/solvers/empty_cache.py | 7 +- .../analyze/query/solvers/series_cache.py | 25 +- .../analyze/query/solvers/solver_config.py | 25 + src/impulse_query_engine/measurement_db.py | 21 + .../model/series/points_in_time_series.py | 34 +- src/impulse_query_engine/schema.py | 20 + src/impulse_reporting/config/config_parser.py | 1 + tests/conftest.py | 31 + .../integration/poi_channel_solve_test.py | 243 +++++ ...default_solver_wide_column_mapping_test.py | 1 + .../solvers/default_solver_wide_only_test.py | 1 + .../query/solvers/solver_config_test.py | 12 +- .../data/basic_narrow_csv/channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_tags.csv | 2 + 28 files changed, 1892 insertions(+), 101 deletions(-) create mode 100644 demos/data/reporting/poi_channels.csv create mode 100644 poi_series_integration.md create mode 100644 tests/impulse_query_engine/integration/poi_channel_solve_test.py diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 5637f583..7846174f 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,13 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME diff --git a/demos/data/reporting/channel_tags.csv b/demos/data/reporting/channel_tags.csv index a35cf010..59185950 100644 --- a/demos/data/reporting/channel_tags.csv +++ b/demos/data/reporting/channel_tags.csv @@ -119,3 +119,39 @@ container_id,channel_id,key,value 3,10,model,Leon 3,10,to_city,RT 3,10,unit,C +1,90,brand,Seat +1,90,channel_name,DTC +1,90,model,Leon +1,90,experiment_id,experiment_4 +1,90,ecu,Engine_ECU +1,90,bus,CAN1 +1,90,code_system,P +1,91,brand,Seat +1,91,channel_name,DTC_count +1,91,model,Leon +1,91,experiment_id,experiment_4 +1,91,ecu,Engine_ECU +2,90,brand,Seat +2,90,channel_name,DTC +2,90,model,Leon +2,90,experiment_id,experiment_4 +2,90,ecu,Engine_ECU +2,90,bus,CAN1 +2,90,code_system,P +2,91,brand,Seat +2,91,channel_name,DTC_count +2,91,model,Leon +2,91,experiment_id,experiment_4 +2,91,ecu,Engine_ECU +3,90,brand,Seat +3,90,channel_name,DTC +3,90,model,Leon +3,90,experiment_id,experiment_4 +3,90,ecu,Body_ECU +3,90,bus,CAN2 +3,90,code_system,U +3,91,brand,Seat +3,91,channel_name,DTC_count +3,91,model,Leon +3,91,experiment_id,experiment_4 +3,91,ecu,Body_ECU diff --git a/demos/data/reporting/poi_channels.csv b/demos/data/reporting/poi_channels.csv new file mode 100644 index 00000000..84357279 --- /dev/null +++ b/demos/data/reporting/poi_channels.csv @@ -0,0 +1,13 @@ +container_id,channel_id,timestamp,value_double,value_string,dtype +1,90,1519629856439000,,P0301,string +1,90,1519631856439000,,P0301,string +1,90,1519633356439000,,P0135,string +1,91,1519629856439000,1.0,,double +1,91,1519631856439000,2.0,,double +1,91,1519633356439000,3.0,,double +2,90,1519756824107000,,P0420,string +2,90,1519758824107000,,P0128,string +2,91,1519756824107000,1.0,,double +2,91,1519758824107000,2.0,,double +3,90,1519926478375000,,U0100,string +3,91,1519926478375000,1.0,,double diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 8c3e5ebd..edc9c41b 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -16,21 +16,21 @@ } }, "source": [ - "# Impulse — Reporting Pipeline Demo\n", + "# Impulse \u2014 Reporting Pipeline Demo\n", "\n", "The **Impulse Framework** is a Python library that enables\n", "automotive and industrial engineers to process, aggregate,\n", "and analyze petabytes of time-series measurement data on\n", - "Databricks — without requiring Apache Spark expertise.\n", + "Databricks \u2014 without requiring Apache Spark expertise.\n", "\n", "It provides **TSAL** (Time Series Analytics Language),\n", "a Pythonic expression language for defining signals,\n", "events, and aggregations.\n", "\n", "**What this notebook builds:**\n", - "A complete reporting pipeline — RPM histograms,\n", + "A complete reporting pipeline \u2014 RPM histograms,\n", "RPM-vs-speed heatmaps, per-distance-bin statistics, and\n", - "channel values sampled at every 10 km milestone — across\n", + "channel values sampled at every 10 km milestone \u2014 across\n", "3 test drives, persisted as a Gold-layer star schema and\n", "visualized inline with matplotlib.\n", "\n", @@ -61,9 +61,9 @@ "\n", "Impulse sits between a governed silver layer and a gold-layer star schema in Unity Catalog and provides three components:\n", "\n", - "- **TSAL (Time Series Analytics Language)** — a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", - "- **Query Engine** — pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", - "- **Aggregations** — domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." + "- **TSAL (Time Series Analytics Language)** \u2014 a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", + "- **Query Engine** \u2014 pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", + "- **Aggregations** \u2014 domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." ] }, { @@ -229,7 +229,7 @@ " (e.g., one test drive)\n", "- **Channel** = one sensor signal within a container\n", " (e.g., Engine RPM), stored as raw\n", - " `(timestamp, value)` samples — the framework\n", + " `(timestamp, value)` samples \u2014 the framework\n", " automatically converts these to intervals on the fly" ] }, @@ -263,7 +263,7 @@ "SILVER = [\n", " \"container_metrics\", \"container_tags\",\n", " \"channel_metrics\", \"channel_tags\",\n", - " \"channels\",\n", + " \"channels\", \"poi_channels\",\n", "]\n", "for t in SILVER:\n", " pdf = pd.read_csv(f\"{csv_dir}/{t}.csv\")\n", @@ -344,13 +344,13 @@ "# 2. Initialize the Report\n", "\n", "The `Report` orchestrator takes a config specifying:\n", - "- **`source`** — Silver layer tables\n", - "- **`unity_sink`** — Gold layer output\n", - "- **`query_engine.solver`** — `DefaultSolver` for\n", + "- **`source`** \u2014 Silver layer tables\n", + "- **`unity_sink`** \u2014 Gold layer output\n", + "- **`query_engine.solver`** \u2014 `DefaultSolver` for\n", " parallel per-container execution\n", - "- **`query_engine.data_type`** — `RAW` for raw\n", + "- **`query_engine.data_type`** \u2014 `RAW` for raw\n", " timestamp data (auto-converted to intervals)\n", - "- **`measurement_dimensions`** — container metadata\n", + "- **`measurement_dimensions`** \u2014 container metadata\n", " to carry into Gold layer" ] }, @@ -391,6 +391,7 @@ " \"container_metrics_table\": f\"{pfx}_container_metrics\",\n", " \"channel_metrics_table\": f\"{pfx}_channel_metrics\",\n", " \"channels_uri\": f\"{pfx}_channels\",\n", + " \"poi_channels_uri\": f\"{pfx}_poi_channels\",\n", " \"container_tags_table\": f\"{pfx}_container_tags\",\n", " \"channel_tags_table\": f\"{pfx}_channel_tags\",\n", " },\n", @@ -440,7 +441,7 @@ "source": [ "# 3. Select Physical Channels\n", "\n", - "Channels are selected by **metadata tags** —\n", + "Channels are selected by **metadata tags** \u2014\n", "no column names, no SQL, no joins.\n", "These are **lazy expressions**: no data is read yet." ] @@ -500,7 +501,7 @@ "# 4. Define Virtual Signals & Events\n", "\n", "**TSAL** uses Python operators to build lazy\n", - "expression trees — no Spark knowledge needed.\n", + "expression trees \u2014 no Spark knowledge needed.\n", "\n", "**Virtual signals** derive from physical channels.\n", "**Events** are time windows where a condition holds." @@ -536,7 +537,7 @@ ")\n", "\n", "# Instant the trip odometer crosses each additional\n", - "# 10 km — a set of points in time, not an interval.\n", + "# 10 km \u2014 a set of points in time, not an interval.\n", "distance_milestones = (distance_km % 10).falling_edges()" ] }, @@ -558,9 +559,9 @@ "source": [ "# 5. Register Events\n", "\n", - "- **BasicEvent** — from a TSAL boolean expression\n", - "- **ContainerEvent** — spans the entire recording\n", - "- **PointsInTimeEvent** — a set of instants (e.g. each 10 km milestone)" + "- **BasicEvent** \u2014 from a TSAL boolean expression\n", + "- **ContainerEvent** \u2014 spans the entire recording\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" ] }, { @@ -628,10 +629,10 @@ "source": [ "# 6. Define Aggregations\n", "\n", - "- **Histogram** — 1D duration-weighted distribution\n", - "- **Histogram2D** — 2D heatmap of two signals\n", - "- **StatisticsAggregator** — min, median, mean, max per event\n", - "- **PointValueAggregator** — channel value sampled at each instant of a points-in-time event" + "- **Histogram** \u2014 1D duration-weighted distribution\n", + "- **Histogram2D** \u2014 2D heatmap of two signals\n", + "- **StatisticsAggregator** \u2014 min, median, mean, max per event\n", + "- **PointValueAggregator** \u2014 channel value sampled at each instant of a points-in-time event" ] }, { @@ -712,7 +713,7 @@ "))\n", "\n", "# Sample Vehicle Speed & Engine RPM at each 10 km\n", - "# milestone — one value per channel per instant.\n", + "# milestone \u2014 one value per channel per instant.\n", "page.add_aggregation(PointValueAggregator(\n", " name=\"values_at_distance_milestones\",\n", " input_expressions=[veh_spd, eng_rpm],\n", @@ -771,8 +772,8 @@ "source": [ "# 7. Compute & Persist\n", "\n", - "- `determine_report()` — parallel execution\n", - "- `persist_results()` — writes star schema" + "- `determine_report()` \u2014 parallel execution\n", + "- `persist_results()` \u2014 writes star schema" ] }, { @@ -819,11 +820,11 @@ "Read the Gold-layer tables back and render the\n", "results inline with **matplotlib**:\n", "\n", - "- **Bar** — RPM histogram\n", - "- **Heatmap** — RPM vs Speed\n", - "- **Table** — per-container statistics\n", - "- **Scatter** — Speed & RPM at each 10 km milestone\n", - " (markers only — values exist only *at* each instant)" + "- **Bar** \u2014 RPM histogram\n", + "- **Heatmap** \u2014 RPM vs Speed\n", + "- **Table** \u2014 per-container statistics\n", + "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", + " (markers only \u2014 values exist only *at* each instant)" ] }, { @@ -846,12 +847,12 @@ "source": [ "import matplotlib.pyplot as plt\n", "\n", - "# ─── Table prefix ───\n", + "# \u2500\u2500\u2500 Table prefix \u2500\u2500\u2500\n", "T = f\"{pfx}\"\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 1. BAR — RPM Histogram (aggregated across all containers)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 1. BAR \u2014 RPM Histogram (aggregated across all containers)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "hist_df = (\n", " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", @@ -869,14 +870,14 @@ "ax.bar(hist_df[\"bin_name\"], hist_df[\"duration_s\"], color=\"steelblue\", edgecolor=\"white\")\n", "ax.set_xlabel(\"Engine RPM bin\")\n", "ax.set_ylabel(\"Duration (s)\")\n", - "ax.set_title(\"RPM Histogram — Duration in Each RPM Band (all containers)\")\n", + "ax.set_title(\"RPM Histogram \u2014 Duration in Each RPM Band (all containers)\")\n", "plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 2. HEATMAP — RPM vs Speed\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 2. HEATMAP \u2014 RPM vs Speed\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "heat_df = (\n", " spark.read.table(f\"{T}_histogram2d_fact\")\n", " .groupBy(\"x_bin_id\", \"y_bin_id\", \"x_bin_name\", \"y_bin_name\",\n", @@ -913,14 +914,14 @@ "ax.set_yticklabels([lbl[1] for lbl in y_labels], fontsize=7)\n", "ax.set_xlabel(\"Engine RPM\")\n", "ax.set_ylabel(\"Vehicle Speed (km/h)\")\n", - "ax.set_title(\"RPM vs Speed Heatmap — Duration (s)\")\n", + "ax.set_title(\"RPM vs Speed Heatmap \u2014 Duration (s)\")\n", "plt.colorbar(im, ax=ax, label=\"Duration (s)\")\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 3. TABLE — Per-container Statistics (container_stats)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 3. TABLE \u2014 Per-container Statistics (container_stats)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "stats_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", @@ -948,9 +949,9 @@ " ),\n", ")\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 4. SCATTER — Speed & RPM at Each 10 km Milestone\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 4. SCATTER \u2014 Speed & RPM at Each 10 km Milestone\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "milestone_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 253c03d9..67c0c089 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -3,6 +3,39 @@ sidebar_label: time_series_expression title: impulse_query_engine.analyze.metadata.time_series_expression --- +## SeriesType + +```python +class SeriesType(StrEnum) +``` + +How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + +``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is +*valid* (reconstructed by an interpolation method, zero-order hold today), +backed by :class:`SampleSeries`. + +``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no +between-point validity, backed by :class:`PointsInTimeSeries`. + + +## PoiValueType + +```python +class PoiValueType(StrEnum) +``` + +The value data type of a POI channel — selects its ``poi_channels`` value + +column and which in-memory :class:`PointsInTimeSeries` variant is built. + +``DOUBLE`` — numeric points (``poi_channels.value_double``); the full +arithmetic / ordering / reduction operator set applies. + +``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); +only sampling and equality apply (see :class:`PointsInTimeSeries`). + + ## TimeSeriesSelector ```python @@ -12,7 +45,10 @@ class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization) #### \_\_init\_\_ ```python -def __init__(expr, uses_alias: bool = False) +def __init__(expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE) ``` Initialize a TimeSeriesSelector. @@ -20,6 +56,18 @@ Initialize a TimeSeriesSelector. **Arguments**: - `expr` (`TagExpression`): Tag expression to select. +- `uses_alias` (`bool`): Whether the channel resolves via the channel-alias table. +- `series_type` (`SeriesType`): How the selected channel's samples are interpreted. ``SAMPLE`` +(default) builds a :class:`SampleSeries` — today's behavior, +unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` +(values valid only at their timestamps); identification / matching is +identical, only the built object and its result dtype differ. This is +the plan-time source of truth for the series type (so ``dtype()`` is +correct for a bare POI selection with no per-channel metadata lookup). +- `value_type` (`PoiValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type +(``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time +typing and string-op gating; validated against the silver +``poi_channels.dtype`` at solve time (assertion contract). #### dtype @@ -31,7 +79,10 @@ Returns the Spark data type. **Returns**: -`pyspark.sql.types.DataType`: Data type (BinaryType). +`pyspark.sql.types.DataType`: ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), +or the value-type-aware ``PointsInTimeSeries.dtype()`` for a +POINTS_IN_TIME selection (``array>`` for numeric, +``array>`` for string). #### deserialize @@ -39,7 +90,11 @@ Returns the Spark data type. def deserialize(d) ``` -Deserialize sample series after collection/toPandas. +Deserialize a SAMPLE result after collection/toPandas. + +POINTS_IN_TIME results are serialized by ``get_data()`` (a plain +``[[t, v], ...]`` list) and need no deserialization, so they are returned +as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. **Arguments**: @@ -47,23 +102,21 @@ Deserialize sample series after collection/toPandas. **Returns**: -`SampleSeries`: Deserialized sample series. +`SampleSeries or Any`: Deserialized sample series (SAMPLE), else *d* unchanged. #### build ```python -def build(cache: SeriesCache) -> SampleSeries +def build(cache: SeriesCache) ``` -Instantiate a SampleSeries from given cache data. +Instantiate the selected series from cache data. -**Arguments**: - -- `cache` (`SeriesCache`): Cache containing time series data. +Resolution is identical regardless of series type — resolve the matching +candidates, take the first ``(container_id, channel_id)``, and let the +cache build the right object. The **data** is authoritative for the built +type: :meth:`TimeSeriesCache.load_blob` returns a -**Returns**: - -`SampleSeries`: Built sample series. #### get\_required\_tag\_exprs diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 40fffeb0..1f5fb82e 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -119,6 +119,40 @@ Create a time series selector for the given channel tags. `TimeSeriesSelector`: Time series selector object. +#### poi\_channel + +```python +def poi_channel(dtype: PoiValueType = PoiValueType.DOUBLE, + **kwargs) -> TimeSeriesSelector +``` + +Create a Points-in-Time (POI) channel selector. + +Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` +from a tag/column match on ``**kwargs`` (e.g. +``poi_channel(channel_name="DTC")``), differing only in that it is stamped +``series_type=POINTS_IN_TIME`` (so it solves to a +:class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` +— a value valid only *at* each timestamp — rather than a ``SampleSeries``) +and carries the declared value ``dtype``. + +Channel *identification* (tag/column match, ``get_selector_expr``, +``required_tags``, ``selector_id``) is identical to :meth:`channel`; only +the built object and its result dtype differ. + +**Arguments**: + +- `dtype` (`PoiValueType`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or +``STRING`` (e.g. DTC codes — only sampling and equality apply). This +declared type drives plan-time result typing and string-op gating; it +is validated against the silver ``poi_channels.dtype`` at solve time +(an actual/declared mismatch raises). +- `**kwargs` (`dict`): Channel tag-value pairs, matched exactly like :meth:`channel`'s. + +**Returns**: + +`TimeSeriesSelector`: A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + #### select ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index 8d65a48d..e16b99cb 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -233,6 +233,42 @@ def value_col() -> str Internal column name for the signal value on the channels table. +#### poi\_timestamp\_col + +```python +def poi_timestamp_col() -> str +``` + +Internal column name for the point timestamp on the poi_channels table. + + +#### poi\_value\_double\_col + +```python +def poi_value_double_col() -> str +``` + +Internal column name for the numeric value on the poi_channels table. + + +#### poi\_value\_string\_col + +```python +def poi_value_string_col() -> str +``` + +Internal column name for the string value on the poi_channels table. + + +#### poi\_dtype\_col + +```python +def poi_dtype_col() -> str +``` + +Internal column name for the per-row value-dtype discriminator on poi_channels. + + #### tag\_key\_col ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index ef8b12ce..bdf55c39 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -24,6 +24,11 @@ A PointsInTimeSeries associates a value to each timestamp. Unlike a SampleSeries a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. +The value type (numeric vs string) is inferred from *values*. An **empty** +series has no values to infer from and therefore defaults to numeric; use +:meth:`empty_string` when an explicitly string-typed empty series is needed +(e.g. plan-time result typing of a bare string-POI selection). + **Arguments**: - `tstarts` (`Sized`): Array-like of time points. @@ -420,9 +425,27 @@ Returns a string representation for debugging. def empty() -> PointsInTimeSeries ``` -Returns an empty PointsInTimeSeries. +Returns an empty (numeric) PointsInTimeSeries. + +**Returns**: + +`PointsInTimeSeries`: Empty numeric PointsInTimeSeries object. + +#### empty\_string + +```python +def empty_string() -> PointsInTimeSeries +``` + +Returns an empty **string-valued** PointsInTimeSeries. + +An empty series has no values to infer a type from, so the constructor +defaults to numeric; this factory forces the string value type. Used for +plan-time result typing of a bare string-POI selection, where the empty +series must report the string ``dtype()`` and reject numeric-only ops +(e.g. ``mean()``) before any data is read. **Returns**: -`PointsInTimeSeries`: Empty PointsInTimeSeries object. +`PointsInTimeSeries`: Empty string-valued PointsInTimeSeries object. diff --git a/poi_series_integration.md b/poi_series_integration.md new file mode 100644 index 00000000..5c975be4 --- /dev/null +++ b/poi_series_integration.md @@ -0,0 +1,896 @@ +--- +sidebar_position: 1 +title: POI Series Integration +--- + +# Design: Integrating Points-in-Time (POI) Series into the Silver Layer + +**Status:** Proposed  ·  **Scope:** `impulse_query_engine` silver-layer +data model + `DefaultSolver` solve stage  ·  **Non-goal:** changing the +6-stage filter pipeline. + +## 1. Summary + +Impulse currently models every channel as a **sample series** — a sequence of +`[tstart, tend)` intervals over which the series is assumed to be **valid** (this +validity is what channel synchronization relies on). It does *not* intrinsically +assume the value is *held constant* over the interval: how a value is reconstructed +within `[tstart, tend)` is an **interpolation** choice. Today the only interpolation +used is **zero-order hold** (the value at `tstart` carries forward), but additional +interpolation methods could be added in the future without changing the underlying +validity model. We want to add a second kind of channel, a **Points-in-Time (POI) +Series**: a list of `(tᵢ, vᵢ)` pairs where each value is defined **only at its +timestamp** and **no assumption of validity (and hence no interpolation) is made +between two consecutive timestamps**. + +The backend model class already exists — +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +— and already implements arithmetic, comparisons, `synchronized` / `synchronized_all`, +and the reducing aggregations (`count`, `sum`, `mean`, `min`, `max`). The +integration work is therefore **not** about series math; it is about: + +1. **Where POI samples live in silver** (a new `poi_channels` table), and +2. **How the solver knows a channel is POI** — table membership (data in + `poi_channels` ⇒ POI) plus the query author's `poi_channel(...)` selector; no + explicit `series_type` column is needed (see [§3.2](#32-discriminator-table-membership-which-table-holds-the-channels-data)), and +3. **How the solve step builds a `PointsInTimeSeries` instead of a `SampleSeries`** + for those channels. + +The central design observation is that the entire metadata **filter pipeline is +already series-type-agnostic**, so POI support drops into the *solve* stage only. + +:::note Terminology + +- **Sample series** — the existing channel type; `[tstart, tend)` intervals over + which the series is *valid*, with values reconstructed by an interpolation method + (zero-order hold today). Backed by `SampleSeries`. +- **POI series** — the new channel type; `(tᵢ, vᵢ)` points valid *only at* their + timestamps, with no between-point validity or interpolation. Backed by + `PointsInTimeSeries`. + +::: + +### 1.1 Motivating example: ECU defect / error codes (DTCs) + +The canonical real-world POI series in vehicle testing is the stream of **defect +codes** (a.k.a. error codes, or **Diagnostic Trouble Codes — DTCs**) emitted by a +vehicle's Electronic Control Units (ECUs). When an ECU's diagnostic monitor detects +a fault — a misfire, a sensor reading out of range, a lost CAN message — it emits a +code at the **instant the fault is registered**. In a test fleet these are captured +off the CAN/UDS bus (e.g. via the `ReadDTCInformation` service, UDS `0x19`) and +logged with the timestamp at which the ECU reported them. + +A DTC event stream is a **textbook POI series**, and specifically a **string-valued** +one: + +- **Event-driven, not continuous.** A code exists *at* the moment the ECU raised it + and says **nothing** about the time between two codes. Interpolating "the value + between two error codes" is meaningless — which is exactly the POI validity model + (no between-point validity), and exactly what the held-over-interval `SampleSeries` + model would get *wrong*. +- **String values.** The standardized code is a short alphanumeric string in the + `P0301` form (1 letter for the system — **P**owertrain / **C**hassis / **B**ody / + **U**network — a generic/OEM digit, a subsystem family digit, and a 2-digit fault + index; e.g. `P0301` = cylinder-1 misfire). This is why POI channels need the + string `value_type` from [§3.4](#34-per-channel-value-dtype-double-vs-string): the + natural analysis is *equality* ("when did `P0301` occur?"), never arithmetic or + ordering on the code — matching the equality-only operator set we implement for + string POI series. + +This use case also motivates **mix-and-match** ([§2](#2-background-why-this-fits-so-cleanly)), +because DTCs are almost always analyzed **together with the continuous signals** +recorded in the same container: + +- **"Freeze-frame"-style analysis.** ECUs snapshot continuous PIDs (engine RPM, + vehicle speed, coolant temperature, …) at the instant a code is set. In Impulse + this is the exact shape of `PointValueAggregator` / a `PointsInTimeEvent`: sample a + `SampleSeries` channel (`Engine_RPM`) **at the timestamps of** a POI channel + (`DTC == "P0301"`). The POI channel supplies the instants; the sample channel + supplies the values valid at those instants — one query, one container, both series + types in the same pandas UDF. +- **Counting / windowing.** "How many `P0301` events occurred while + `Engine_RPM > 4000`?" combines a string-POI equality filter with an interval + derived from a sample series — again both series types in one expression. + +Sketched in the query API this design proposes ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), +the string DTC channel is selected with the dedicated `poi_channel(...)` method +and its `dtype`, and mixes freely with an ordinary `channel(...)` sample selection: + +```python +dtc = query.poi_channel(channel_name="DTC", dtype="string") # string POI series +rpm = query.channel(channel_name="Engine_RPM") # sample series + +# "freeze-frame": RPM at the instants DTC == "P0301" +rpm.where(dtc == "P0301") + +# equality is the only comparator defined on a string POI series (§3.4) +``` + +:::note Timestamp caveat (informative) + +DTCs do not universally carry an absolute wall-clock timestamp in the ECU fault +memory — the reliable instant usually comes from the logger/gateway that timestamps +the event when it reads the code (GPS/NTP-synced), and any per-DTC snapshot/extended +records are OEM-dependent. For Impulse this is an **ingestion** concern: whatever +timestamp the silver pipeline lands on `poi_channels.timestamp` is the instant the +engine treats as the point's `tᵢ`. It does not affect the data-model or solver +design below. + +::: + +## 2. Background: why this fits so cleanly + +The [`DefaultSolver` filter pipeline](../references/query_engine/query_solvers.md) +runs six stages, but only ever passes **identity + selector metadata** between +them: + +``` +filter_container_tags → filter_container_metrics → filter_channel_tags → +filter_channel_metrics → (alias resolution) → solve +``` + +Every stage up to `solve` produces at most +`(container_id, channel_id, selector_ids)` (plus optional unit columns). **None of +these stages read `tstart` / `tend` / `value` or make any interval-validity or +interpolation assumption.** The validity-and-interpolation semantics enter the +system in exactly one place: + +- `DefaultSolver.solve` reads the `channels` table, joins it to the channel-match + frame, and runs a grouped-map UDF (`_solve_udf`). +- Inside the UDF, `TimeSeriesCache.load_blob(...)` constructs a **`SampleSeries`** + from the `(ts, te, val)` columns. +- `TimeSeriesSelector.build(cache)` calls `cache.load_blob(...)` and returns that + `SampleSeries` to the expression tree. + +So a channel becomes a `SampleSeries` at `TimeSeriesCache.load_blob`, and nowhere +else. If we can make that one call return a `PointsInTimeSeries` for POI channels, +the rest of the engine — expression evaluation, events, aggregations — already +works, because `PointsInTimeSeries` and `SampleSeries` share the operator and +synchronization protocol, and `SampleSeries.where(PointsInTime)` / +`PointsInTimeSeries.plane_sweep` already bridge the two representations. + +**Mixing is the common case, and it is a per-container concern.** POI and sample +channels live in the **same containers**, and users routinely combine them in one +expression (e.g. `poi_channel - sample_channel`). Cross-type alignment +(`synchronized`) happens **inside** the per-container pandas UDF, on the in-memory +series objects. That imposes a hard requirement: **both series types for a container +must be present in the same UDF invocation.** A design that solved sample and POI +channels in two *separate* UDFs and unioned the results would break mixing — each +UDF would see only half of a container's channels and could not evaluate a +cross-type expression. The design below therefore feeds **one unified pandas frame +per container** (sample + POI channel data together) into a **single** grouped-map +UDF, and a unified cache builds the correct series type per channel. + +```mermaid +flowchart TB + subgraph pipeline["Filter pipeline (UNCHANGED — series-type-agnostic)"] + direction LR + A[container tags] --> B[container metrics] --> C[channel tags] --> D[channel metrics] + end + D -->|"(container_id, channel_id, selector_ids, series_type)"| J + + subgraph solve["solve stage (the ONLY place that touches series semantics)"] + direction TB + RS["read channels
(SAMPLE rows)"] --> J + RP["read poi_channels
(POI rows)"] --> J + J["union sample + POI sample data
keyed by (container_id, channel_id)
→ ONE frame per container"] + J --> U["grouped-map UDF, grouped by container_id
(both series types in the same pandas frame)"] + U --> C2["unified cache builds per channel:
SampleSeries (valid over interval; ZOH today)
or PointsInTimeSeries (valid only at points)"] + C2 --> EV["evaluate expression tree
cross-type ops align via synchronized"] + end + EV --> OUT([one wide row per container]) +``` + +## 3. Chosen design + +### 3.1 Storage: a separate `poi_channels` table + +POI samples are stored in a **new silver table `poi_channels`**, parallel to +`channels` but carrying a single timestamp (no derived `tend`, because a POI point +has no notion of a validity interval) and **two typed value columns** plus a +per-row `dtype` discriminator, since a POI value may be numeric **or** a string: + +| Column | Type | Nullable | Description | +|----------------|----------|----------|-------------------------------------------------------------------| +| `container_id` | `long` | No | Parent container identifier (join key). | +| `channel_id` | `int` | No | Channel identifier. | +| `timestamp` | `long` | No | Point timestamp (microseconds). | +| `value_double` | `double` | Yes | Value at this timestamp **when `dtype = double`**; else null. | +| `value_string` | `string` | Yes | Value at this timestamp **when `dtype = string`**; else null. | +| `dtype` | `string` | No | Value data type: `double` or `string`. Selects the value column. | + +For any given row exactly one of `value_double` / `value_string` is populated, +chosen by `dtype`. `dtype` is expected to be **constant per +`(container_id, channel_id)`** — a channel is either a numeric POI channel or a +string POI channel, not a mix (see [§3.4](#34-per-channel-value-dtype-double-vs-string)). + +`container_id` follows the same +[type rules as the rest of the silver layer](../data_model/silver_layer_schema.md) +— it may be `long` / `int` / `string`, but must be **consistent across all silver +tables** since the engine joins on it. This matches the CLAUDE.md invariant that +`container_id` / `channel_id` types are derived dynamically and never hardcoded. + +**Why a separate table rather than reusing `channels`:** + +- **No semantic overloading of `tend`.** The `channels` RLE format treats a + trailing zero-duration `[t, t)` row as a *closed endpoint* of a sample series (an + interval of validity that has collapsed to a single instant). Reusing that row + shape for a *whole* POI channel would require every + reader (the solve UDF, the RLE/interval encoders, `SampleSeries` construction) to + disambiguate "closed endpoint of a sample series" from "a genuine point". A + dedicated table keeps the two data shapes physically and semantically distinct. +- **Cleaner ingestion contract.** Producers write POI points as `(timestamp, value)` + with no obligation to synthesize a `tend`, which they cannot do correctly for POI + data anyway. +- **Minimal disturbance to the sample-series path.** The existing `channels` read + and RLE/interval encoding are untouched, and a `SAMPLE` channel still builds the + identical `SampleSeries`. The cache does gain a per-channel series-type dispatch + (required so sample and POI channels can be mixed in one UDF — see + [§4.3](#43-one-unified-per-container-frame-one-udf-one-dispatching-cache)), but the + sample branch's behavior is unchanged. + +The cost is a new configured table + a new read path + a branch in the solve +prelude — all localized to `DefaultSolver.solve` / `MeasurementDB` (see §4). + +### 3.2 Discriminator: a `series_type` column on `channel_metrics` + +:::note Implemented differently — see [§9](#9-aspects-which-differ-from-the-design) +The `series_type` column described below was **not** added. Table membership +(`channels` vs `poi_channels`) is the discriminator instead. See [§9](#9-aspects-which-differ-from-the-design). +::: + +A channel is marked POI by a **new `series_type` column on `channel_metrics`**: + +| Column | Type | Nullable | Description | +|---------------|----------|----------|--------------------------------------------------------------------| +| `series_type` | `string` | Yes | `SAMPLE` (default) or `POINTS_IN_TIME`. Null/absent ⇒ `SAMPLE`. | + +Design points: + +- **Backward compatible.** Existing tables without the column, or with `NULL`, + resolve to `SAMPLE`, so every current deployment behaves exactly as today. +- **Rides the pipeline as pass-through metadata.** `channel_metrics` is already + read in `filter_channel_metrics`; `series_type` is just one more column carried + on the channel-match rows through to `solve`. It participates in **no** filtering + decision. +- **Not `value_type`.** `channel_metrics.value_type` already exists but describes + the *value's data type* (`double`, `int`, …). Overloading it to also encode + *series semantics* would conflate two orthogonal concepts and is rejected. A new, + purpose-specific column keeps the discriminator explicit and self-documenting. +- **Introduce a `SeriesType` enum** (mirroring `RawEncoder`) so the string literals + live in one place and are referenced by `SolverConfig.series_type_col` / + the solve branch rather than being sprinkled as bare strings. + +`series_type` is added to `SolverConfig` as an internal column name property +(`series_type_col`, default `"series_type"`), so a physical layout that names the +column differently maps it via `channel_metrics.column_name_mapping` exactly like +every other column. + +### 3.3 Data model after the change + +```mermaid +erDiagram + container_metrics { + long container_id PK + } + channel_metrics { + long container_id FK + int channel_id FK + string series_type "SAMPLE | POINTS_IN_TIME (null ⇒ SAMPLE)" + } + channels { + long container_id FK + int channel_id FK + long tstart + long tend + double value + } + poi_channels { + long container_id FK + int channel_id FK + long timestamp + double value_double "when dtype = double" + string value_string "when dtype = string" + string dtype "double | string" + } + + container_metrics ||--o{ channel_metrics : container_id + channel_metrics ||--o{ channels : "SAMPLE channels" + channel_metrics ||--o{ poi_channels : "POINTS_IN_TIME channels" +``` + +A given `(container_id, channel_id)` has its samples in **exactly one** of +`channels` or `poi_channels`, selected by its `series_type` row in +`channel_metrics`. + +### 3.4 Per-channel value dtype: double vs string + +The `poi_channels.dtype` column determines which value column +(`value_double` / `value_string`) carries the point value. We treat `dtype` as a +**per-channel** property: all rows of a `(container_id, channel_id)` share one +`dtype`. This keeps a channel's value type stable, matches how measurement channels +behave in practice, and lets the solve step pick the value column **once** per +channel rather than per row. + +The two dtypes are **not** symmetric, because the backend model represents them +differently. `PointsInTimeSeries` **cannot represent string values today** — its +constructor hardcodes `np.array(values, dtype=np.float64)`, which would coerce +strings to `NaN`. We close this gap by extending the **single** +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +class to hold values of either kind, rather than adding a second class. + +**Chosen model change — dual value arrays + a `value_type` property:** + +- **Keep the existing `float64` value array** for numeric values (unchanged; today's + numeric behavior is preserved bit-for-bit). +- **Add a second value array of dtype `object`** to hold string values. + *(Implemented differently — a single value array whose type is inferred at + construction; see [§9](#9-aspects-which-differ-from-the-design).)* +- **Add a `value_type` property on the class** distinguishing a **numeric** from a + **string** POI series. This is the single source of truth for which value array is + populated and which operations are legal. (Constructors/factories set it; a + numeric series leaves the object array empty and vice-versa.) +- **Spark `dtype()` becomes `value_type`-aware:** `ArrayType(ArrayType(DoubleType))` + for numeric (unchanged), `ArrayType(ArrayType(StringType))` for string. + +**Operations on a string POI series (this iteration):** + +- **Only the equality comparator (`==`) is implemented.** It matches the numeric + behavior — synchronize on shared timestamps, compare values, return the + `PointsInTime` where values are equal — but over string values. + *(Implemented more permissively — both `==` and `!=` are supported for strings; + see [§9](#9-aspects-which-differ-from-the-design).)* +- **All other comparators (`<`, `<=`, `>`, `>=`) return a + `NotImplementedError`** for a string series, as do the numeric-only reductions and + arithmetic (`sum`, `mean`, `min`, `max`, `+`, `-`, `*`, `/`). These raise a clear, + explicit error rather than silently coercing to `NaN`. +- Value-type-independent operations remain valid regardless of `value_type`: + `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and the + timestamp side of `synchronized`. + +:::note "series type" appears on three distinct axes — keep them straight + +| Where | Values | Meaning | +|-------|--------|---------| +| table membership (`channels` / `poi_channels`) | sample vs POI | Which table holds the channel's data — *this* is the sample-vs-POI discriminator (no `series_type` column; see [§9](#9-aspects-which-differ-from-the-design)). | +| `poi_channels.dtype` (silver column) | `double` / `string` | A POI channel's value type — selects `value_double` vs `value_string`. | +| `PointsInTimeSeries.value_type` (class property) | numeric / string | Which in-memory value array is active and which operations are legal. | + +The middle and bottom rows are the same distinction on two sides of the Arrow +boundary: `poi_channels.dtype` on a channel becomes `PointsInTimeSeries.value_type` +on the object the cache builds for it. + +::: + +The **selectable operations are gated by `value_type`** so that, e.g., +`string_poi.mean()` fails up front (via `evaluation_type()` — see [§4.4](#44-result-typing)) +rather than producing `NaN`. + +:::note Scope check + +String POI support is the one part of this design that requires touching the +backend model (`PointsInTimeSeries`). Everything else — storage, discriminator, +pipeline, solve branch — is additive. If string POI is not needed in the first +iteration, the numeric (`double`) path can ship alone: the solver simply routes +only `dtype = double` channels and rejects (or ignores, per config) `string` +channels until the model work lands. + +::: + +### 3.5 Example: tag & metric entries for DTC POI channels + +Concrete rows for the [DTC example](#11-motivating-example-ecu-defect--error-codes-dtcs), +on an existing recording `container_id = 1`. Two POI channels are added on +`channel_id`s not used by any sample channel in that container: a **string** DTC-code +channel (`channel_id = 90`) and a **numeric** fault-occurrence-count channel +(`channel_id = 91`). + +#### Channel level — where POI-specific entries naturally live + +**Channel selection metadata.** In the EAV layout these are `channel_tags` rows +(`container_id, channel_id, key, value`); in the wide layout the same facts are +columns on `channel_metrics`. A DTC channel is selected by its `channel_name` and +described by ECU/bus context: + +| container_id | channel_id | key | value | +|--------------|------------|----------------|--------------| +| 1 | 90 | `channel_name` | `DTC` | +| 1 | 90 | `ecu` | `Engine_ECU` | +| 1 | 90 | `bus` | `CAN1` | +| 1 | 90 | `code_system` | `P` (powertrain) | +| 1 | 91 | `channel_name` | `DTC_count` | +| 1 | 91 | `ecu` | `Engine_ECU` | + +**Channel metrics** (`channel_metrics`). The **new `series_type`** marks the channel +as POI; the **existing `value_type`** records the value data type. Crucially, the +numeric statistic columns behave differently by value type — they are **undefined +(null) for a string POI channel**, and meaningful (computed over the point values, +**unweighted** — there are no durations) for a numeric one: + +| Column | DTC string channel (90) | DTC count numeric channel (91) | Notes | +|----------------|-------------------------|--------------------------------|-------| +| `series_type` | `POINTS_IN_TIME` | `POINTS_IN_TIME` | new discriminator (§3.2) | +| `value_type` | `STRING` | `DOUBLE` | pre-existing data-type column | +| `channel_name` | `DTC` | `DTC_count` | selection key (wide layout) | +| `sample_count` | `3` (three events) | `3` | number of points | +| `begin_s`/`end_s` | first/last event time | first/last event time | point extent, not a validity span | +| `min`/`max`/`mean`/`std` | **null** | computed over point values | undefined for strings; unweighted for numeric POI | +| `pz1`/`pz10`/`pz90`/`pz99` | **null** | optional | percentiles undefined for strings | +| `nan_ratio` | **null** | **null** | duration-weighted → N/A for POI | + +The per-row **`dtype`** (`string` / `double`) lives on `poi_channels`, not here (§3.1); +`series_type` on `channel_metrics` is what routes the channel to `poi_channels`. + +#### Container level — optional summaries for pre-filtering + +A container is a whole recording and owns **both** sample and POI channels, so +container-level tags/metrics are **not** POI-specific — the usual `vehicle_key`, +`brand`, `model`, `project` entries are unchanged. What POI *optionally* adds here is +**summary metadata that lets you pre-filter containers** without scanning +`poi_channels` (the same role the percentile columns play for sample channels): + +EAV `container_tags` (`container_id, key, value`): + +| container_id | key | value | Purpose | +|--------------|------------------|-------------|---------| +| 1 | `vehicle_key` | `Seat_Leon` | existing — unchanged | +| 1 | `has_dtc` | `true` | optional — "recordings that logged any fault" | +| 1 | `ecu_sw_version` | `4.11.2` | optional — correlate faults with firmware | + +Wide `container_metrics` can carry the analogous optional column +`num_dtc_events = 3` for the same pre-filtering purpose. + +These container-level additions are **purely optional and additive**: omit them and +POI channels still work; add them only to enable "find recordings where a `P0301` +occurred"-style container filters before the channel stage. A query like +`query.havingTag(has_dtc="true")` then narrows containers exactly as any other +container tag does — no POI-specific pipeline behavior. + + +## 4. Implementation plan + +The change is localized. Nothing in stages 1–5 of the pipeline changes. + +### 4.1 Config & schema + +1. `SolverConfig`: add `poi_channels: TableConfig`, add the `series_type_col` + property (`"series_type"`), and add a `poi_channels_uri` slot to + `MeasurementDBConfig` (+ `for_unity_catalog` / `for_debug` wiring, mirroring + `channels_uri`). `poi_channels_uri = None` means "no POI channels configured". +2. `MeasurementDB.poi_channels(spark)` reader, mirroring `channels(...)`. +3. `schema.py`: add a reference `POI_CHANNELS_SCHEMA` (`container_id`, `channel_id`, + `timestamp`, `value_double`, `value_string`, `dtype`) and add `series_type` to + `CHANNEL_METRICS`. As documented in CLAUDE.md these are **reference** schemas, + not enforced on read. +4. Add a `SeriesType` StrEnum (`SAMPLE`, `POINTS_IN_TIME`) next to `RawEncoder`, and + a `PoiValueType` StrEnum (`double`, `string`) for the per-row `dtype`. +5. Add `SolverConfig` internal-name properties for the new POI columns + (`poi_timestamp_col`, `poi_value_double_col`, `poi_value_string_col`, + `poi_dtype_col`) so physical layouts remap them via + `poi_channels.column_name_mapping` like every other table. +6. Extend `SolverConfig.col_map` (the short-key → column-name map handed to the UDF + cache, today `cid/ch/ts/te/val/conv`) with `series_type`, `value_string`, and + `dtype` keys so the unified cache (§4.3) can locate them in the pandas frame. +7. Add two optional fields to `TimeSeriesSelector` (`series_type`, `value_type`), + defaulting to `SAMPLE` / numeric so existing `channel(...)` selectors are + unchanged, and add `QueryBuilder.poi_channel(*, dtype=PoiValueType.double, + **kwargs)` (see [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + +### 4.2 Query API and carrying the discriminators to solve + +#### `QueryBuilder.poi_channel(...)` + +POI channels are selected through a dedicated **`poi_channel(...)` factory method** +on `QueryBuilder`, parallel to the existing `channel(...)` / `channel_with_alias(...)`: + +```python +def poi_channel(self, *, dtype: PoiValueType = PoiValueType.double, **kwargs) -> TimeSeriesSelector: + # same tag/column matching as channel(...) — builds the selector expr from **kwargs + return TimeSeriesSelector(expr, series_type=POINTS_IN_TIME, value_type=dtype) +``` + +Design points: + +- **No new selector class.** `poi_channel` returns the **same `TimeSeriesSelector`** + that `channel(...)` returns; channel *identification* (tag/column match, + `get_selector_expr`, `required_tags`, `selector_id`, the direct/aliased split) is + identical for POI and sample channels, so there is nothing to override. The method + is a **factory**, not a subclass — it just stamps the selector with its + `series_type` (`POINTS_IN_TIME`) and the caller-declared value `dtype`. +- **Explicit intent at the call site.** `query.poi_channel(channel_name="DTC")` + reads as "this is an event stream, not a signal," and gives POI-only knobs + (the `dtype`) a natural home. `dtype` defaults to `double`, so the common numeric + case stays terse; a string DTC channel is `poi_channel(channel_name="DTC", dtype=string)`. +- **The selector now carries `series_type` + `value_type`.** `TimeSeriesSelector` + gains two optional fields (defaulting to `SAMPLE` / numeric so `channel(...)` is + unchanged). This makes the selector the **plan-time** source of truth for the + series type — which is what simplifies result typing (see [§4.4](#44-result-typing)): + `evaluation_type()` / `dtype()` and the string-op gating work **without** any + pre-pipeline `channel_metrics` lookup, and `string_poi.mean()` can be rejected at + **build time** before Spark is involved. + +:::caution Declared `dtype` is validated against the data, not trusted over it + +The user-declared `dtype` and the silver data are **two sources that must agree**. +The contract is **assertion, not authority**: + +> The check validates against the **data itself**, not a `channel_metrics.series_type` +> column (which was dropped — see [§9](#9-aspects-which-differ-from-the-design)): a POI +> point row carries a null `tend`, so a `poi_channel(...)` that resolves to +> interval-shaped rows is a SAMPLE channel, and an all-null value column exposes a +> declared/actual `dtype` mismatch. + +- The declared `series_type` / `dtype` drive **plan-time** typing and op-gating. +- At **solve time** the data remains authoritative: if the resolved channel's actual + shape **disagrees** with what the selector declared, the solver **raises a clear + error** (mirroring the existing unit-conversion conflict check), rather than silently + reading the wrong value column or overriding the data. + +This keeps the ergonomic win (no plan-time lookup, early validation) without letting +a wrong declaration silently mis-read a channel (e.g. a `dtype=double` hint on a +string channel yielding all-null `value_double`). + +::: + +#### Carrying the discriminators through the pipeline + +`filter_channel_metrics` already reads and column-maps `channel_metrics`. Include +`series_type` in the projected channel-match columns (defaulting null → `SAMPLE` +via `F.coalesce`). It travels alongside `selector_ids` with no effect on any +filter, exactly like the existing per-channel metadata. This solve-time +`series_type` (and, for POI, `dtype`) is what the **assertion check above** +validates the selector's declared values against. + +### 4.3 One unified per-container frame, one UDF, one dispatching cache + +Because sample and POI channels share containers and are mixed in a single +expression, they **must be solved together in one grouped-map UDF per container** +(see the requirement established in [§2](#2-background-why-this-fits-so-cleanly)). +The design keeps the existing single-UDF shape and makes the *cache* series-type +aware, rather than forking the UDF. + +**Step 1 — normalize both sample sources into one Spark frame.** In +`_prepare_channels_join`, read and column-map **both** tables and project them into +a common superset schema keyed by `(container_id, channel_id)`, carrying a +`series_type` discriminator (and, for POI, `dtype`): + +| Column | SAMPLE row source | POI row source | +|----------------|-----------------------|---------------------------------------| +| `container_id` | `channels` | `poi_channels` | +| `channel_id` | `channels` | `poi_channels` | +| `series_type` | `SAMPLE` | `POINTS_IN_TIME` | +| `tstart` | `channels.tstart` | `poi_channels.timestamp` | +| `tend` | `channels.tend` | `null` (POI has no validity interval) | +| `value` | `channels.value` | `poi_channels.value_double` | +| `value_string` | `null` | `poi_channels.value_string` | +| `dtype` | `null` (⇒ numeric) | `poi_channels.dtype` | + +`unionByName` the two projections into a single DataFrame, join it to the +channel-match frame on `(container_id, channel_id)`, then — exactly as today — +`groupBy(container_id).apply(udf)`. Only channels that survived the filter pipeline +are shipped, so the union stays small. A container's sample and POI rows now land in +the **same** pandas frame. + +**Step 2 — a unified cache that dispatches per channel.** Generalize +`TimeSeriesCache` (or add a `UnifiedSeriesCache` that subsumes it) so `load_blob` +inspects the channel slice's `series_type` and builds the right object: + +- `series_type == SAMPLE` → `SampleSeries(tstart, tend, value)` (today's behavior, + unchanged). +- `series_type == POINTS_IN_TIME` and `dtype == double` → numeric + `PointsInTimeSeries(tstart, value)` (the POI timestamp lives in the `tstart` + column of the unified frame). +- `series_type == POINTS_IN_TIME` and `dtype == string` → the string point series + from [§3.4](#34-per-channel-value-dtype-double-vs-string), built from + `(tstart, value_string)`. + +The cache keeps the same `(cid, ch) → (start, stop)` range-index over the sorted +frame; the only change is which columns each slice reads and which class it +instantiates. Because `series_type` and `dtype` are constant per channel, the +dispatch is decided **once** per `(cid, ch)` slice, not per row. + +**Step 3 — expression evaluation is unchanged.** `TimeSeriesSelector.build(cache)` +still just calls `cache.load_blob(...)`; it now transparently gets a `SampleSeries` +or a point series. A mixed expression such as `poi_channel - sample_channel` is +evaluated on the two in-memory objects, and `PointsInTimeSeries._apply_basic_op` +already handles the cross-type case by aligning against the `SampleSeries` at the +POI timestamps via `synchronized`. **No new math and no second UDF.** + +The `series_type` / `dtype` discriminators are carried the same pass-through way as +the existing per-channel metadata (they originate on `channel_metrics` / +`poi_channels`; see [§8](#8-open-questions)), so both the cache and the result-typing +step (§4.4) know each channel's kind without scanning its data. + +:::note Why not two UDFs? + +Splitting SAMPLE and POI into two grouped-map UDFs and unioning their **outputs** +would be simpler to write but is **incorrect** for the common mix-and-match case: +each UDF would receive only a subset of a container's channels, so an expression +referencing one channel of each type could not be evaluated — one operand would +always be missing from that UDF's frame. Unifying the **input** frame and keeping a +single UDF is what makes cross-type expressions work. + +::: + +### 4.4 Result typing + +`QueryBuilder._determine_result_objects_dtypes` builds each selection against an +`EmptyTimeSeriesCache` to learn its result `dtype`. Today `EmptyTimeSeriesCache.load_blob` +always returns an empty `SampleSeries`, so a bare POI selection would be mistyped +as `BinaryType` (the `SampleSeries` serialization dtype) instead of +`PointsInTimeSeries.dtype()` (`ArrayType(ArrayType(DoubleType))`). + +**Because the selector now carries its own `series_type` / `value_type` +([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), this resolves with +no plan-time metadata lookup.** `EmptyTimeSeriesCache.load_blob` simply consults the +calling selector and returns an empty series of the matching kind: + +- a `SAMPLE` selector → empty `SampleSeries` (today's behavior); +- a numeric POI selector → empty numeric `PointsInTimeSeries`; +- a string POI selector → empty `PointsInTimeSeries` with `value_type = string`. + +`evaluation_type()` / `dtype()` are then correct for bare POI selections and for +expressions whose output type depends on the input type — and the string-op gating +fires **at build time**: `string_poi.mean()` builds an empty string point series +whose `mean()` raises `NotImplementedError`, so the selection is rejected up front +rather than producing a silent `NaN`, before Spark is involved. + +This removes the earlier need to pre-resolve each selector's type from +`channel_metrics` and inject it into the empty cache: the declared type on the +selector *is* the plan-time source. (The silver metadata still has the final say at +solve time via the [§4.2 assertion check](#42-query-api-and-carrying-the-discriminators-to-solve).) +It also mirrors how `PointsInTimeEvent` and `PointValueAggregator` already validate +`evaluation_type()` up front, so the mechanism is consistent with existing code. + +### 4.5 `PointsInTimeSeries` model change + +The one backend-model change (see [§3.4](#34-per-channel-value-dtype-double-vs-string)): + +- Add a second value array (dtype `object`) alongside the existing `float64` array, + and a `value_type` property (numeric / string) selecting which is active. +- Constructors/factories set `value_type`: the numeric path keeps today's + `np.array(values, dtype=np.float64)`; the string path stores values as an `object` + array and leaves the numeric array empty. +- Make `dtype()` return `ArrayType(ArrayType(StringType))` when `value_type` is + string (numeric unchanged). +- Implement **`__eq__` for string series** (synchronize on timestamps → compare + string values → `PointsInTime`). Have `__ne__`, `__lt__`, `__le__`, `__gt__`, + `__ge__`, the arithmetic operators, and the numeric reductions (`sum`, `mean`, + `min`, `max`) **raise `NotImplementedError`** when `value_type` is string. +- Leave `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and + the timestamp handling in `synchronized` value-type-independent (they already are). + +### 4.6 Extend the existing test dataset with DTC POI channels + +Rather than build a bespoke POI fixture, **extend the existing session-scoped silver +dataset** so POI channels live alongside the current sample channels in the **same +containers** — this is what exercises the mix-and-match path (§4.3) end to end and +mirrors the [DTC motivating example](#11-motivating-example-ecu-defect--error-codes-dtcs). +The guiding constraint is **additive, non-destructive**: every existing test must +keep passing untouched. + +The `setup_basic_db` fixture (autouse, session-scoped) loads +`tests/unit/data/basic_narrow_csv/` into `spark_catalog.silver.*`. Use the concrete +rows from [§3.5](#35-example-tag--metric-entries-for-dtc-poi-channels) (DTC string +channel `channel_id = 90`, numeric count channel `channel_id = 91` on +`container_id = 1`) as the fixture data. The plan: + +1. **New `poi_channels` data file.** Add + `basic_narrow_csv/poi_channels.csv` with + `container_id, channel_id, timestamp, value_double, value_string, dtype` and a + couple of **DTC channels** on **existing** `container_id`s (e.g. a `DTC` string + channel with points like `(t₁, "P0301")`, `(t₂, "P0420")`, and a numeric POI + channel such as a fault-occurrence counter). Choose `channel_id`s **not already + used** by that container in `channel_data.csv` so the two sample sources stay + disjoint per the design invariant (a channel lives in exactly one of + `channels` / `poi_channels`). +2. **Append POI rows to `channel_metrics.csv`.** Add one row per new POI channel + carrying the new `series_type = POINTS_IN_TIME` column. **Backfill existing rows + with `series_type = SAMPLE`** (or leave blank and rely on the null ⇒ `SAMPLE` + default — pick one and be consistent). Existing sample channels are unaffected. +3. **Load `poi_channels` in the fixture.** Extend `setup_basic_db` to read the new + CSV and write `spark_catalog.silver.poi_channels`, and add its slot to the + `MeasurementDBConfig` used by the basic-db fixtures (`poi_channels_uri`). Because + `poi_channels_uri` defaults to `None`, **any db config that does not opt in is + unchanged**, so unrelated fixtures/tests see no difference. +4. **EAV + wide tag/metric parity.** So POI channels are *selectable* the same way + in both channel-selection modes: + - **EAV fixtures** (`setup_narrow_db`, `unit_test_csv/`): append POI rows to + `1_channel_tags.csv` (e.g. `channel_name = "DTC"`) and `1_channel_metrics.csv`, + plus any container-level tags/metrics needed, so a + `query.poi_channel(channel_name="DTC", dtype="string")` resolves the POI channel + through the pivot path (identification is identical to `channel(...)`; only the + selector's declared `series_type` / `value_type` differ — [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + - **Wide fixtures** (`basic_narrow_csv`): the `channel_name` column already on + `channel_metrics` covers direct selection; just ensure the appended POI rows + carry a distinct `channel_name` (e.g. `"DTC"`). + +:::caution Two different columns both once called "value type" + +`basic_narrow_csv/channel_metrics.csv` **already** has a `value_type` column holding +values like `DOUBLE` (and the EAV `1_channel_metrics.csv` has `numerical`). That is +the **pre-existing** per-channel data-type column and is **not** the discriminator +this design adds. Keep them separate: + +- **existing `channel_metrics.value_type`** — untouched; describes the value data + type and is not read by the solver for routing. +- **new `channel_metrics.series_type`** — `SAMPLE` / `POINTS_IN_TIME`; routes to + `channels` vs `poi_channels` (§3.2). +- **new `poi_channels.dtype`** — `double` / `string`; selects `value_double` / + `value_string` (§3.4). + +Do **not** overload the existing `value_type` column for either new purpose — the +column names in the fixtures must stay distinct, and existing tests that read +`value_type` must be left as-is. + +::: + +**Regression guard.** Run the full existing suite after extending the fixtures and +confirm it is green *before* adding POI-specific tests (§7). Because the changes are +purely additive — new file, appended rows with a defaulting column, an opt-in table +slot — no existing assertion (row counts, computed means, dimension contents) should +move. If any does, the extension was not additive and must be corrected. + +## 5. What explicitly does **not** change + +- **The 6-stage filter pipeline.** `filter_container_tags` → + `filter_container_metrics` → `filter_channel_tags` → `filter_channel_metrics` → + alias resolution are untouched. POI channels are identified by the *same* + `TimeSeriesSelector` class, tag/column matching, and tag/metric filters as sample + channels — `poi_channel(...)` is a factory over the same selector, not a new + selection path ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). +- **`channels` table and RLE/interval encoders.** The sample-series read and + raw→interval encoding are untouched. +- **The sample-series *behavior* in the cache.** `TimeSeriesCache` gains a + per-channel dispatch (§4.3), but for a `SAMPLE` channel it builds the exact same + `SampleSeries(tstart, tend, value)` as today — the sample path's semantics and + output are unchanged. (This is a behavior guarantee, not a "no code changed" + claim: the cache does gain POI-aware branching.) +- **The single grouped-map UDF per container.** The solve stage still groups by + `container_id` and applies one UDF; POI does **not** add a second UDF or a + post-hoc union of two result sets. The input frame is widened to carry both series + types, not the execution model. +- **Persistence / gold layer.** Aggregations over POI series already reduce to + scalars (`mean`, `sum`, `count`, …) or `PointsInTime` events, which the existing + fact/dimension tables already accept (`PointsInTimeEvent`, `PointValueAggregator`). +- **`PointsInTimeSeries` for numeric (`double`) channels.** No new methods needed; + it already implements the full operator/sync/aggregation protocol for `float64` + values. (String POI is the exception — it requires the model change in + [§3.4](#34-per-channel-value-dtype-double-vs-string).) +- **`SampleSeries` interpolation semantics.** This design does not change how + sample-series values are reconstructed within `[tstart, tend)`. Zero-order hold + remains the only interpolation today; adding further interpolation methods later + is an **orthogonal** effort. The distinction that matters for POI is *validity* + (does a value exist between two timestamps at all?), not *which* interpolation is + applied where validity holds. + +## 6. Alternatives considered + +| Alternative | Why not chosen | +|-------------|----------------| +| **Store POI in `channels` with `tend == tstart`** | Overloads the "closed endpoint" meaning of zero-duration rows; forces every reader/encoder to disambiguate a whole POI channel from a sample-series endpoint. | +| **Store POI in the RAW `channels` (timestamp, value) format + a skip-encoding flag** | Couples POI to RAW mode and to the raw→interval encoder; a channel's storage shape would depend on an unrelated `data_type` setting. | +| **Overload the existing `value_type` column as the discriminator** | Conflates value *data type* with *series semantics*; two orthogonal concerns in one column, harder to reason about and to validate. | +| **A new dedicated POI solver class** | Unnecessary — the filter pipeline is shared and identical; only `load_blob` differs. A per-channel branch inside `DefaultSolver.solve` is far less code than a parallel solver. | +| **Two grouped-map UDFs (one SAMPLE, one POI), union the outputs** | **Incorrect** for the common mix-and-match case: each UDF sees only a subset of a container's channels, so an expression combining a POI and a sample channel (e.g. `poi - sample`) has a missing operand. Cross-type `synchronized` must run on both in-memory series inside **one** UDF. | + +## 7. Testing strategy + +Following the repo's fixture-reuse convention (CLAUDE.md → *Testing patterns*). +The POI tests run against the **extended shared dataset from [§4.6](#46-extend-the-existing-test-dataset-with-dtc-poi-channels)** +(DTC channels added to the existing `spark_catalog.silver.*` fixtures) rather than a +throwaway db, so they cover the real read path and the mix-and-match case: + +- Assert on **real computed values**, not row counts: e.g. a numeric POI `mean()` + equals the unweighted mean of the point values (contrast with the duration-weighted + `SampleSeries.mean()`, whose weighting follows from interval validity), proving the + between-point validity is genuinely absent. +- A **string POI** test: `query.poi_channel(channel_name="DTC", dtype="string")` + builds a `PointsInTimeSeries` with `value_type = string`; the **equality comparator** + (`== "P0301"` → `PointsInTime` on matching timestamps) and value-type-independent ops + (`count`, `to_points_in_time`, point sampling) work, while every **other comparator** + (`!=`, `<`, `<=`, `>`, `>=`), the arithmetic operators, and the numeric reductions + (`mean`, `sum`, `min`, `max`) raise `NotImplementedError` — asserted both directly on + the series object and, for a reduction inside a selection, at `evaluation_type()` + **build time** (not as a silent `NaN`, and before Spark runs). +- A **mix-and-match test (the primary correctness case)**: a single container owning + both a SAMPLE channel and a numeric POI channel, selected with `query.channel(...)` + and `query.poi_channel(...)` respectively, with **one expression referencing both** + (`rpm.where(dtc == "P0301")`, and `poi - sample`). This asserts both series land in + the *same* per-container pandas frame, are built by the unified cache, and align via + `synchronized` — the behavior a two-UDF design would break. Assert the computed + values, not just that it runs. +- A **declared-vs-actual `dtype` assertion test** ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)): + `query.poi_channel(channel_name="DTC", dtype="double")` on a channel whose silver + `dtype` is `string` (or a `poi_channel` on a `SAMPLE` channel) raises a clear error + at solve time — the data stays authoritative, the wrong declaration is not silently + honored. +- A backward-compat test: a `channel_metrics` with no `series_type` column still + solves as SAMPLE, and existing `channel(...)` selections are unaffected by the new + optional selector fields. + +## 8. Open questions + +- **Should `series_type` be validated against the presence of data in the matching + table?** (e.g. a POI-marked channel with rows only in `channels`.) Proposed: + no hard validation initially; document that the marker is authoritative and the + non-matching table is not read for that channel. +- **~~Where should the POI value `dtype` be resolved for planning?~~ (Resolved.)** + The user declares `dtype` on `query.poi_channel(...)` and the selector carries it + ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), so plan-time + result typing (§4.4) needs **no** pre-pipeline `poi_channels` / `channel_metrics` + scan. The silver `poi_channels.dtype` remains authoritative at solve time and is + validated against the declared value (assertion contract). *Remaining sub-question:* + should the engine also support **inferring** `dtype` when the user omits it (rather + than defaulting to `double`) — e.g. a cheap `distinct` on `channel_metrics` — for + callers who prefer not to declare it? Proposed: keep the explicit `double` default + for now; add inference only if a concrete need appears. +- **Enforcing the constant-`dtype` invariant.** A channel is entirely numeric or + entirely string — `dtype` is constant per `(container_id, channel_id)` by + contract. This is a settled invariant, not an open question; the only decision is + whether to *defend* it. Proposed: an optional validate-and-raise (like the + unit-conversion conflict check) that flags any channel carrying more than one + distinct `dtype`, so a malformed ingest fails loudly instead of picking an + arbitrary value column. +- **String value column when scaling.** If more non-numeric dtypes appear later + (e.g. `bool`, `int`), revisit whether a typed-column-per-dtype layout still scales + or whether a single `value` string column + cast is preferable. +- **Calculated channels producing POI output.** `solve_calculated_channels` emits a + narrow `[container_id, channel_id, tstart, tend, value]` frame. Emitting a POI + *calculated* channel would need a narrow POI shape (`timestamp, value`). Deferred — + out of scope for ingesting POI *input* series. + +## 9. Aspects which differ from the design + +A few things landed differently than sections 3–4 describe. The shipped code is the +source of truth; those sections are left as the original proposal, and each spot that +changed points here. None of these change what the feature does — they mostly remove +machinery the design added that turned out to be unnecessary once the selector became +the source of truth for a channel's series type. + +### 9.1 No `series_type` column on `channel_metrics` (§3.2) + +The design added a `series_type` marker to `channel_metrics` so the solver could tell a +POI channel from a sample channel. We dropped it. A channel's data lives in exactly one +of `channels` or `poi_channels`, so **which table it comes from already tells us the +series type** — the extra column was redundant, and nothing ever read it at solve time. +Today the only way to get a `PointsInTimeSeries` is to read from `poi_channels`, so the +table membership is a complete answer. + +### 9.2 One value array, type inferred at construction (§3.4, §4.5) + +The design proposed keeping the numeric `float64` array and adding a *second* `object` +array for strings, with a `value_type` property choosing between them. In practice +`PointsInTimeSeries` keeps a **single** value array and infers whether it's string or +numeric from the values at construction time (an `_is_string` flag). It's less +bookkeeping — there's no pair of arrays to keep in sync, one always empty — and it +reads more naturally: you build the series from whatever values you have and it figures +out its own type. An explicit `empty_string()` factory covers the one case inference +can't (an empty series has nothing to infer from). + +### 9.3 String POI also supports `!=` (§3.4, §4.5) + +The design limited string POI series to equality (`==`) and had `!=` raise alongside +the ordering and arithmetic operators. We kept `!=` too. + +### 9.4 The declared-vs-actual check reads the data shape, not a marker (§4.2) + +The design validated the selector's declared `series_type` / `dtype` against the +`channel_metrics.series_type` column. With that column gone (9.1), the solve-time check +instead looks at the **data it resolved to** + +### 9.5 Series-type dispatch is driven by the selector, not a per-row column (§4.3) + +The design's solve stage stamped `series_type` (and `dtype`) onto every channel-data +row so the cache could inspect each slice. Since the selector already knows its own +type, we pass that into `load_blob` instead and drop the per-row markers from the frame +that crosses into the pandas UDF. Only `value_string` still rides along, because that's +real data a string channel needs, not a discriminator. The result is the same object +per channel with a bit less shipped across the Arrow boundary. + +### 9.6 Enum placement (§4.1) + +Minor: the design suggested putting `SeriesType` next to `RawEncoder` in +`solver_config.py`. It lives in `time_series_expression.py` instead, next to +`TimeSeriesSelector` (which carries it) and alongside the new `PoiValueType` enum. That's +where the selector-as-source-of-truth logic reads most naturally. diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 0977dc32..87bd166a 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -4,18 +4,50 @@ import operator import zlib from collections.abc import Callable, Iterable +from enum import StrEnum from typing import TYPE_CHECKING, Any import pyspark.sql.types as T import impulse_query_engine.util as U from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries if TYPE_CHECKING: from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache +class SeriesType(StrEnum): + """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + + ``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is + *valid* (reconstructed by an interpolation method, zero-order hold today), + backed by :class:`SampleSeries`. + + ``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no + between-point validity, backed by :class:`PointsInTimeSeries`. + """ + + SAMPLE = "SAMPLE" + POINTS_IN_TIME = "POINTS_IN_TIME" + + +class PoiValueType(StrEnum): + """The value data type of a POI channel — selects its ``poi_channels`` value + column and which in-memory :class:`PointsInTimeSeries` variant is built. + + ``DOUBLE`` — numeric points (``poi_channels.value_double``); the full + arithmetic / ordering / reduction operator set applies. + + ``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); + only sampling and equality apply (see :class:`PointsInTimeSeries`). + """ + + DOUBLE = "double" + STRING = "string" + + class RequiresDeserialization: pass @@ -619,7 +651,13 @@ def from_dict(obj: dict) -> TimeSeriesExpression: class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization): - def __init__(self, expr, uses_alias: bool = False): + def __init__( + self, + expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE, + ): """ Initialize a TimeSeriesSelector. @@ -627,18 +665,48 @@ def __init__(self, expr, uses_alias: bool = False): ---------- expr : TagExpression Tag expression to select. + uses_alias : bool, optional + Whether the channel resolves via the channel-alias table. + series_type : SeriesType, optional + How the selected channel's samples are interpreted. ``SAMPLE`` + (default) builds a :class:`SampleSeries` — today's behavior, + unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` + (values valid only at their timestamps); identification / matching is + identical, only the built object and its result dtype differ. This is + the plan-time source of truth for the series type (so ``dtype()`` is + correct for a bare POI selection with no per-channel metadata lookup). + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selection, the declared value data type + (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time + typing and string-op gating; validated against the silver + ``poi_channels.dtype`` at solve time (assertion contract). """ self._expr = expr self._uses_alias = uses_alias + self._series_type = series_type + self._value_type = value_type TimeSeriesExpression.__init__(self, is_single_signal=True) @property def uses_alias(self) -> bool: return self._uses_alias + @property + def series_type(self) -> SeriesType: + return self._series_type + + @property + def value_type(self) -> PoiValueType: + return self._value_type + @property def selector_id(self) -> int: - return zlib.crc32(str(self._expr).encode()) + # Include series_type so a SAMPLE and a POINTS_IN_TIME selection of the + # same tag expression resolve as distinct channels. SAMPLE keeps the + # historical id (bare ``str(expr)`` hash) for backward compatibility. + if self._series_type is SeriesType.SAMPLE: + return zlib.crc32(str(self._expr).encode()) + return zlib.crc32(f"{self._series_type}|{self._expr}".encode()) def dtype(self): """ @@ -647,13 +715,33 @@ def dtype(self): Returns ------- pyspark.sql.types.DataType - Data type (BinaryType). + ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), + or the value-type-aware ``PointsInTimeSeries.dtype()`` for a + POINTS_IN_TIME selection (``array>`` for numeric, + ``array>`` for string). """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time().dtype() return T.BinaryType() + def _empty_points_in_time(self) -> PointsInTimeSeries: + """Empty POI series carrying this selector's declared value type. + + A string selector must build a string-typed empty series so ``dtype()`` + and the string-op gating (e.g. ``.mean()`` raising) reflect the declared + type before any data is read. + """ + if self._value_type is PoiValueType.STRING: + return PointsInTimeSeries.empty_string() + return PointsInTimeSeries.empty() + def deserialize(self, d): """ - Deserialize sample series after collection/toPandas. + Deserialize a SAMPLE result after collection/toPandas. + + POINTS_IN_TIME results are serialized by ``get_data()`` (a plain + ``[[t, v], ...]`` list) and need no deserialization, so they are returned + as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. Parameters ---------- @@ -662,14 +750,26 @@ def deserialize(self, d): Returns ------- - SampleSeries - Deserialized sample series. + SampleSeries or Any + Deserialized sample series (SAMPLE), else *d* unchanged. """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return d return SampleSeries.deserialize(d) - def build(self, cache: SeriesCache) -> SampleSeries: + def build(self, cache: SeriesCache): """ - Instantiate a SampleSeries from given cache data. + Instantiate the selected series from cache data. + + Resolution is identical regardless of series type — resolve the matching + candidates, take the first ``(container_id, channel_id)``, and let the + cache build the right object. The **data** is authoritative for the built + type: :meth:`TimeSeriesCache.load_blob` returns a + :class:`PointsInTimeSeries` for a ``POINTS_IN_TIME`` slice and a + :class:`SampleSeries` otherwise. The selector's own :attr:`series_type` / + :attr:`value_type` are used only for **plan-time** typing (:meth:`dtype` + against an empty cache), so a bare POI selection types correctly and a + string-only op is rejected before Spark runs. Parameters ---------- @@ -678,16 +778,25 @@ def build(self, cache: SeriesCache) -> SampleSeries: Returns ------- - SampleSeries - Built sample series. + SampleSeries or PointsInTimeSeries """ candidates = cache.resolve(self) if len(candidates) == 0: + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time() return SampleSeries.empty() # TODO: select candidate mid = candidates.container_id.iloc[0] cid = candidates.channel_id.iloc[0] - return cache.load_blob(mid, cid, uses_alias=self.uses_alias) + # The selector is the source of truth for the series type: pass it to the + # cache so load_blob builds the right object without a per-row discriminator. + return cache.load_blob( + mid, + cid, + uses_alias=self.uses_alias, + series_type=self._series_type, + value_type=self._value_type, + ) def get_required_tag_exprs(self) -> set[TagExpression]: """ @@ -765,6 +874,8 @@ def as_dict(self) -> dict[str, Any]: obj["type"] = U.name_of(TimeSeriesSelector) obj["expr"] = self._expr.as_dict() obj["uses_alias"] = self._uses_alias + obj["series_type"] = str(self._series_type) + obj["value_type"] = str(self._value_type) return obj @staticmethod @@ -783,7 +894,14 @@ def from_dict(obj: dict): Selector instance. """ expr = TimeSeriesExpression.from_dict(obj["expr"]) - m = TimeSeriesSelector(expr, uses_alias=obj.get("uses_alias", False)) + # Default to SAMPLE / DOUBLE so selectors serialized before POI support + # (no series_type / value_type keys) deserialize unchanged. + m = TimeSeriesSelector( + expr, + uses_alias=obj.get("uses_alias", False), + series_type=SeriesType(obj.get("series_type", SeriesType.SAMPLE)), + value_type=PoiValueType(obj.get("value_type", PoiValueType.DOUBLE)), + ) if "alias" in obj and obj["alias"] is not None: m.alias(obj["alias"]) return m diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 26ac47e3..a97b4653 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -7,7 +7,9 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricSelector from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, RequiresDeserialization, + SeriesType, TimeSeriesExpression, TimeSeriesSelector, ) @@ -161,6 +163,48 @@ def channel_with_alias(self, **kwargs) -> TimeSeriesSelector: expr = expr & (TagSelector(k) == str(arg)) return TimeSeriesSelector(expr, uses_alias=True) + def poi_channel( + self, dtype: PoiValueType = PoiValueType.DOUBLE, **kwargs + ) -> TimeSeriesSelector: + """ + Create a Points-in-Time (POI) channel selector. + + Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` + from a tag/column match on ``**kwargs`` (e.g. + ``poi_channel(channel_name="DTC")``), differing only in that it is stamped + ``series_type=POINTS_IN_TIME`` (so it solves to a + :class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` + — a value valid only *at* each timestamp — rather than a ``SampleSeries``) + and carries the declared value ``dtype``. + + Channel *identification* (tag/column match, ``get_selector_expr``, + ``required_tags``, ``selector_id``) is identical to :meth:`channel`; only + the built object and its result dtype differ. + + Parameters + ---------- + dtype : PoiValueType, optional + The POI channel's value data type: ``DOUBLE`` (default, numeric) or + ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + declared type drives plan-time result typing and string-op gating; it + is validated against the silver ``poi_channels.dtype`` at solve time + (an actual/declared mismatch raises). + **kwargs : dict + Channel tag-value pairs, matched exactly like :meth:`channel`'s. + + Returns + ------- + TimeSeriesSelector + A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + """ + expr = None + for k, arg in kwargs.items(): + if not expr: + expr = TagSelector(k) == str(arg) + else: + expr = expr & (TagSelector(k) == str(arg)) + return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + def select(self, *args) -> Self: """ Set the selection expressions for the query. diff --git a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py index d2782de2..72ba79e5 100644 --- a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py @@ -50,10 +50,15 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.df) return self.df[idx] - def load_blob(self, container_id, channel_id, uses_alias: bool = False): + def load_blob( + self, container_id, channel_id, uses_alias: bool = False, series_type=None, value_type=None + ): """ Load a time series blob from disk. + ``series_type`` / ``value_type`` are accepted for interface compatibility + with :class:`SeriesCache`; this blob cache serves only SAMPLE series. + Parameters ---------- container_id : Any diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 02f300fd..e56eb95a 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -11,6 +11,11 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, +) +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries from .query_solver import QuerySolver @@ -43,7 +48,11 @@ def __init__(self, pdf, col_map: dict[str, str]): col_map : dict[str, str] Mapping with keys ``"cid"``, ``"ch"``, ``"ts"``, ``"te"``, ``"val"``, ``"conv"`` to the actual column names in *pdf*. The - ``"conv"`` column is optional in *pdf*. + ``"conv"`` column is optional in *pdf*. For a POI (``POINTS_IN_TIME``) + selector, :meth:`load_blob` builds a :class:`PointsInTimeSeries` — the + **selector** (not a per-row column) chooses the series type; the + ``"value_string"`` key names the string value column that a string POI + slice reads. """ self._cid_col = col_map["cid"] self._ch_col = col_map["ch"] @@ -52,6 +61,9 @@ def __init__(self, pdf, col_map: dict[str, str]): self._val_col = col_map["val"] self._conv_col = col_map.get("conv") self._has_conversion = self._conv_col is not None and self._conv_col in pdf.columns + # String POI slices read their value from this column; series-type dispatch + # is driven by the selector passed to load_blob, not a per-row marker. + self._value_string_col = col_map.get("value_string") # *pdf* holds channel data for a whole container, so avoid creating unnecessary copies of the data. meta_cols = [ @@ -98,13 +110,20 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.mdf) return self.mdf[idx] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Load a time series blob from the DataFrame. + The **calling selector** chooses the series type (via *series_type* / + *value_type*), so no per-row discriminator column is needed: a + ``POINTS_IN_TIME`` selector yields a :class:`PointsInTimeSeries` (string- + valued when *value_type* is ``STRING``, else numeric), otherwise a + :class:`SampleSeries`. The declared type is validated against the silver + metadata in the solve prelude, so the data stays authoritative. + When the underlying *pdf* carries a conversion-factor column (the column named by ``col_map["conv"]``) **and** the caller is an - aliased selector (``uses_alias=True``), the returned values are + aliased selector (``uses_alias=True``), the returned SAMPLE values are multiplied by that factor. Direct selectors on the same physical channel always receive raw values — unit conversion is a property of the alias, not of the channel. @@ -118,14 +137,28 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional ``True`` when the calling selector resolved via channel_mapping. Gates the per-channel conversion factor; defaults to ``False``. + series_type : SeriesType, optional + The calling selector's series type; ``POINTS_IN_TIME`` builds a + :class:`PointsInTimeSeries`. ``None`` (default) => SAMPLE. + value_type : PoiValueType, optional + For a POI selector, its declared value type; ``STRING`` reads the + string value column, otherwise the numeric one. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries """ lo, hi = self._ranges.get((mid, cid), (0, 0)) s = self.pdf.iloc[lo:hi] + + if series_type == SeriesType.POINTS_IN_TIME: + self._assert_poi_data(s, value_type) + if value_type == PoiValueType.STRING: + # value_string is a populated string column, so the constructor + # infers the string value type from it. + return PointsInTimeSeries(s[self._ts_col], s[self._value_string_col]) + return PointsInTimeSeries(s[self._ts_col], s[self._val_col]) + values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: factor = s[self._conv_col].iloc[0] @@ -133,6 +166,52 @@ def load_blob(self, mid, cid, uses_alias: bool = False): values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) + def _assert_poi_data(self, s, value_type) -> None: + """Validate a POI selector against the data it resolved to. + + The selector drives series-type dispatch, but the silver data stays + authoritative: a ``poi_channel(...)`` selector must land on genuine POI + rows. POI rows carry a null ``tend`` (a point has no validity interval), + whereas SAMPLE rows always carry a real ``tend`` (non-nullable in + ``channels``); so a non-null ``tend`` on a POI-declared slice means the + selector was pointed at a SAMPLE channel. A ``STRING`` declaration + additionally requires a populated ``value_string``. Either mismatch raises + rather than silently reading the wrong column (mirrors the unit-conversion + conflict check). + """ + if len(s) == 0: + return + if pd.notna(s[self._te_col].iloc[0]): + raise ValueError( + "POI channel series-type mismatch: poi_channel(...) resolved to a SAMPLE " + "channel (its rows carry a validity interval). Use channel(...) for SAMPLE " + "channels and poi_channel(...) for POINTS_IN_TIME channels." + ) + + has_string_col = self._value_string_col is not None and self._value_string_col in s.columns + string_all_null = has_string_col and s[self._value_string_col].isna().all() + double_all_null = s[self._val_col].isna().all() + + if value_type == PoiValueType.STRING: + # A string POI channel must carry string values; all-null means the + # channel is actually numeric (declared the wrong dtype). + if not has_string_col or string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a channel " + "with no string values (it is a numeric POI channel). Pass dtype=double to " + "poi_channel(...)." + ) + else: + # A numeric POI channel must carry numeric values; all-null numeric + # with populated string values means the channel is actually a string + # channel (declared the wrong dtype). + if double_all_null and has_string_col and not string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a channel " + "whose numeric values are all null (it is a string POI channel). Pass " + "dtype=string to poi_channel(...)." + ) + class DefaultSolver(QuerySolver): """ @@ -1026,6 +1105,15 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra self.config.value_col, ) + # POI channel data is unioned in AFTER RLE encoding above, so its + # zero-duration points are never run-length merged. The inner join to + # channels_df below drops any POI rows whose channel was not selected, so + # unioning whenever a poi_channels table is configured is correct (a + # pure-SAMPLE query simply matches no POI channel_ids). Which object each + # channel builds is decided by the selector (passed to load_blob), not a + # per-row marker — SAMPLE rows just lack value_string. + q = self._union_poi_channel_data(query, q) + joined_df = q.join( F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col], @@ -1033,6 +1121,35 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra container_count = channels_df.select(self.config.container_id_col).distinct().count() return q, joined_df, container_count + def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: + """Union POI channel-data rows into the (already-encoded) channel-data frame. + + Reads ``poi_channels``, column-maps it, and projects it into the SAMPLE + channel-data superset — POI ``timestamp`` becomes ``tstart`` (``tend`` + **null**, since a point has no validity interval, which is also the signal + the cache validates a POI selector against), ``value_double`` becomes the + numeric ``value`` column, and ``value_string`` rides alongside for a string + POI channel. No per-row ``series_type`` / ``dtype`` marker is shipped: the + selector drives series-type dispatch in :meth:`TimeSeriesCache.load_blob`. + Returns *channels_q* unchanged when no ``poi_channels`` table is configured. + """ + db = query.db + if not (hasattr(db, "has_poi_channels") and db.has_poi_channels()): + return channels_q + + cfg = self.config + poi = db.poi_channels(self.spark) + poi = self._apply_column_mapping(poi, cfg.poi_channels.column_name_mapping) + poi_proj = poi.select( + F.col(cfg.container_id_col), + F.col(cfg.channel_id_col), + F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), + F.lit(None).cast(T.LongType()).alias(cfg.tend_col), + F.col(cfg.poi_value_double_col).alias(cfg.value_col), + F.col(cfg.poi_value_string_col).alias(cfg.poi_value_string_col), + ) + return channels_q.unionByName(poi_proj, allowMissingColumns=True) + def _apply_grouped_map(self, joined_df, container_count, schema, solve_udf) -> DataFrame: """Run *solve_udf* per container, or return an empty frame when none match.""" if container_count == 0: diff --git a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py index db348fc6..d0d0016b 100644 --- a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py @@ -25,7 +25,7 @@ def resolve(self, selection): """ return [] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Return an empty SampleSeries for any container and channel ID. @@ -38,6 +38,11 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional Unused by this cache; accepted for interface compatibility with :class:`SeriesCache`. + series_type, value_type : optional + Accepted for interface compatibility. The empty-series typing for a + POI selector is handled by ``TimeSeriesSelector.build`` (its length-0 + branch), which returns the correctly typed empty series without + reaching this method. Returns ------- diff --git a/src/impulse_query_engine/analyze/query/solvers/series_cache.py b/src/impulse_query_engine/analyze/query/solvers/series_cache.py index 8f8ca8e6..aed9d810 100644 --- a/src/impulse_query_engine/analyze/query/solvers/series_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/series_cache.py @@ -27,7 +27,14 @@ def resolve(self, selection) -> pd.DataFrame: pass @abstractmethod - def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: + def load_blob( + self, + mid, + cid, + uses_alias: bool = False, + series_type=None, + value_type=None, + ) -> SampleSeries: """ Resolve given mid and cid to a series. @@ -44,10 +51,22 @@ def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: conversion factor when this is ``True``, so a direct selector on the same physical channel always returns raw values. Defaults to ``False`` (direct / no-conversion semantics). + series_type : SeriesType, optional + The calling selector's series type. The selector — not a per-row + data column — is the source of truth for which object to build: + ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`, otherwise a + :class:`SampleSeries`. ``None`` (default) means SAMPLE, so callers + that predate POI are unchanged. + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selector, its declared value type + (``DOUBLE`` / ``STRING``) — selects the numeric vs string value + column. Ignored for SAMPLE. The declared type is validated against + the silver metadata in the solve prelude, so the data stays + authoritative. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries + The loaded series object. """ pass diff --git a/src/impulse_query_engine/analyze/query/solvers/solver_config.py b/src/impulse_query_engine/analyze/query/solvers/solver_config.py index 0ca71138..0eb11227 100644 --- a/src/impulse_query_engine/analyze/query/solvers/solver_config.py +++ b/src/impulse_query_engine/analyze/query/solvers/solver_config.py @@ -143,6 +143,7 @@ class SolverConfig(BaseModel): channel_metrics: TableConfig = TableConfig() channel_mapping: ChannelMappingConfig = ChannelMappingConfig() channels: TableConfig = TableConfig() + poi_channels: TableConfig = TableConfig() unit_conversion: TableConfig = TableConfig() # ------------------------------------------------------------------ @@ -231,6 +232,26 @@ def value_col(self) -> str: """Internal column name for the signal value on the channels table.""" return "value" + @property + def poi_timestamp_col(self) -> str: + """Internal column name for the point timestamp on the poi_channels table.""" + return "timestamp" + + @property + def poi_value_double_col(self) -> str: + """Internal column name for the numeric value on the poi_channels table.""" + return "value_double" + + @property + def poi_value_string_col(self) -> str: + """Internal column name for the string value on the poi_channels table.""" + return "value_string" + + @property + def poi_dtype_col(self) -> str: + """Internal column name for the per-row value-dtype discriminator on poi_channels.""" + return "dtype" + @property def tag_key_col(self) -> str: """Internal column name for the attribute key on the container_tags (EAV) table.""" @@ -379,4 +400,8 @@ def col_map(self) -> dict[str, str]: "te": self.tend_col, "val": self.value_col, "conv": self.conversion_factor_col, + # String POI slices read their value from this column. Series-type + # dispatch is driven by the selector (passed to load_blob), so no + # per-row series_type / dtype marker column is needed in the frame. + "value_string": self.poi_value_string_col, } diff --git a/src/impulse_query_engine/measurement_db.py b/src/impulse_query_engine/measurement_db.py index c0ba27ca..3ebdcded 100644 --- a/src/impulse_query_engine/measurement_db.py +++ b/src/impulse_query_engine/measurement_db.py @@ -14,6 +14,7 @@ def __init__( channel_tags_table=None, channel_metrics_table=None, channels_uri=None, + poi_channels_uri=None, channel_mapping_table=None, unit_conversion_table=None, table_locations: str = "external_locations", @@ -23,6 +24,9 @@ def __init__( self.channel_tags_table = channel_tags_table self.channel_metrics_table = channel_metrics_table self.channels_uri = channels_uri + # Optional Points-in-Time (POI) channel-data table. ``None`` means no POI + # channels are configured, so POI-unaware deployments are unchanged. + self.poi_channels_uri = poi_channels_uri self.channel_mapping_table = channel_mapping_table self.unit_conversion_table = unit_conversion_table self.table_locations = table_locations @@ -34,6 +38,7 @@ def for_unity_catalog( core_schema_name: str = "core", channel_mapping_table: str | None = None, unit_conversion_table: str | None = None, + poi_channels_uri: str | None = None, ): return MeasurementDBConfig( container_tags_table=f"{catalog_name}.{core_schema_name}.container_tags", @@ -41,6 +46,7 @@ def for_unity_catalog( channel_tags_table=f"{catalog_name}.{core_schema_name}.channel_tags", channel_metrics_table=f"{catalog_name}.{core_schema_name}.channel_metrics", channels_uri=f"{catalog_name}.{core_schema_name}.channels", + poi_channels_uri=poi_channels_uri, channel_mapping_table=channel_mapping_table, unit_conversion_table=unit_conversion_table, table_locations="unity_catalog", @@ -58,6 +64,7 @@ def for_debug(debug_tables): "channel_metrics" if "channel_metrics" in debug_tables else None ), channels_uri="channels" if "channels" in debug_tables else None, + poi_channels_uri="poi_channels" if "poi_channels" in debug_tables else None, channel_mapping_table=( "channel_mapping" if "channel_mapping" in debug_tables else None ), @@ -103,6 +110,20 @@ def channel_metrics(self, spark) -> DataFrame: def channels(self, spark) -> DataFrame: return self._read_table(spark, self.config.channels_uri) + def has_poi_channels(self) -> bool: + """Whether a Points-in-Time (POI) channel-data table is configured.""" + return getattr(self.config, "poi_channels_uri", None) is not None + + def poi_channels(self, spark) -> DataFrame: + """Read the Points-in-Time (POI) channel-data table. + + Parallel to :meth:`channels`. Raises if no ``poi_channels_uri`` is + configured — callers should gate on :meth:`has_poi_channels` first. + """ + if not self.has_poi_channels(): + raise ValueError("poi_channels_uri is not configured") + return self._read_table(spark, self.config.poi_channels_uri) + def channel_mapping(self, spark) -> DataFrame: if self.config.channel_mapping_table is None: raise ValueError("channel_mapping_table is not configured") diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 192a73db..44d8dcd0 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -53,6 +53,11 @@ def __init__(self, tstarts: Sized, values: Sized): a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. + The value type (numeric vs string) is inferred from *values*. An **empty** + series has no values to infer from and therefore defaults to numeric; use + :meth:`empty_string` when an explicitly string-typed empty series is needed + (e.g. plan-time result typing of a bare string-POI selection). + Parameters ---------- tstarts : Sized @@ -65,8 +70,6 @@ def __init__(self, tstarts: Sized, values: Sized): # string-valued series support sampling (``synchronized`` / ``.where``) # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering # and numeric reductions are rejected (see the ``@_numeric_only`` methods). - # An empty series has no observed value type, so it defaults to numeric - # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") if self._is_string: @@ -604,11 +607,34 @@ def __repr__(self) -> str: @staticmethod def empty() -> PointsInTimeSeries: """ - Returns an empty PointsInTimeSeries. + Returns an empty (numeric) PointsInTimeSeries. Returns ------- PointsInTimeSeries - Empty PointsInTimeSeries object. + Empty numeric PointsInTimeSeries object. """ return PointsInTimeSeries([], []) + + @staticmethod + def empty_string() -> PointsInTimeSeries: + """ + Returns an empty **string-valued** PointsInTimeSeries. + + An empty series has no values to infer a type from, so the constructor + defaults to numeric; this factory forces the string value type. Used for + plan-time result typing of a bare string-POI selection, where the empty + series must report the string ``dtype()`` and reject numeric-only ops + (e.g. ``mean()``) before any data is read. + + Returns + ------- + PointsInTimeSeries + Empty string-valued PointsInTimeSeries object. + """ + # A single-element object array makes the constructor infer string, then + # slice back to empty so no value is retained. + series = PointsInTimeSeries([], []) + series._is_string = True + series.values = np.asarray([], dtype=object) + return series diff --git a/src/impulse_query_engine/schema.py b/src/impulse_query_engine/schema.py index 8323182e..f64465bc 100644 --- a/src/impulse_query_engine/schema.py +++ b/src/impulse_query_engine/schema.py @@ -52,6 +52,26 @@ ] ) +# Points-in-Time (POI) channel samples: a value defined only *at* its timestamp +# (no derived tend / validity interval). Two typed value columns plus a per-row +# dtype discriminator, since a POI value may be numeric or a string; exactly one of +# value_double / value_string is populated per row, selected by dtype. +# +# A channel is a POI channel iff its data lives here rather than in ``channels`` — +# table membership *is* the series-type discriminator, so no ``series_type`` column +# is needed on ``channel_metrics``. A given (container_id, channel_id) lives in +# exactly one of ``channels`` / ``poi_channels``. +POI_CHANNELS_SCHEMA = T.StructType( + [ + T.StructField("container_id", T.LongType(), nullable=False), + T.StructField("channel_id", T.IntegerType(), nullable=False), + T.StructField("timestamp", T.LongType(), nullable=False), + T.StructField("value_double", T.DoubleType()), + T.StructField("value_string", T.StringType()), + T.StructField("dtype", T.StringType(), nullable=False), + ] +) + CHANNELS_SCHEMA = T.StructType( [ T.StructField("container_id", T.LongType(), nullable=False), diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index f8e3d0b8..84f49157 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -151,6 +151,7 @@ class Source(BaseModel): channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None + #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/conftest.py b/tests/conftest.py index 0ea87313..d5713299 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,7 @@ def basic_narrow_db(spark, mock_workspace_client) -> MeasurementDB: tables["container_metrics"] = spark.read.table("spark_catalog.silver.container_metrics") tables["channel_metrics"] = spark.read.table("spark_catalog.silver.channel_metrics") tables["channels"] = spark.read.table("spark_catalog.silver.channels") + tables["poi_channels"] = spark.read.table("spark_catalog.silver.poi_channels") cfg = MeasurementDBConfig.for_debug(tables) return MeasurementDB(cfg, ws=mock_workspace_client) @@ -96,6 +97,20 @@ def setup_narrow_db(spark): ), schema=S.CHANNELS_SCHEMA, ) + poi_channels = spark.createDataFrame( + pd.read_csv( + f"{base_path}/tests/unit/data/unit_test_csv/1_poi_channels.csv", + dtype={ + "container_id": np.int64, + "channel_id": np.int32, + "timestamp": np.longlong, + "value_double": np.float64, + "value_string": "object", + "dtype": "object", + }, + ), + schema=S.POI_CHANNELS_SCHEMA, + ) container_tags.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.container_tags" @@ -112,6 +127,9 @@ def setup_narrow_db(spark): channels.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.channels" ) + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver_narrow_db.poi_channels" + ) @pytest.fixture(scope="session", autouse=True) @@ -131,11 +149,20 @@ def setup_basic_db(spark): container_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/container_metrics.csv" channel_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_metrics.csv" channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_data.csv" + poi_channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/poi_channels.csv" options = {"header": "True", "delimiter": ",", "inferSchema": "True"} container_metrics = spark.read.options(**options).csv(container_metric_path) channel_metrics = spark.read.options(**options).csv(channel_metric_path) channels = spark.read.options(**options).csv(channels_path) + # POI channel data: explicit schema so empty value columns keep their nullable + # typed shape (value_double double / value_string string) rather than being + # inferred as all-null strings. + poi_channels = ( + spark.read.schema(S.POI_CHANNELS_SCHEMA) + .options(header="True", delimiter=",") + .csv(poi_channels_path) + ) container_metrics.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver.container_metrics" @@ -150,6 +177,9 @@ def setup_basic_db(spark): "spark_catalog.silver.channel_metrics" ) channels.write.format("delta").mode("overwrite").saveAsTable("spark_catalog.silver.channels") + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver.poi_channels" + ) @pytest.fixture(scope="session") @@ -236,6 +266,7 @@ def narrow_db(spark, setup_narrow_db, mock_workspace_client) -> MeasurementDB: "spark_catalog.silver_narrow_db.channel_metrics" ) debug_tables["channels"] = spark.read.table("spark_catalog.silver_narrow_db.channels") + debug_tables["poi_channels"] = spark.read.table("spark_catalog.silver_narrow_db.poi_channels") cfg = MeasurementDBConfig.for_debug(debug_tables) return MeasurementDB(cfg, ws=mock_workspace_client) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py new file mode 100644 index 00000000..8b032ffc --- /dev/null +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -0,0 +1,243 @@ +"""End-to-end integration tests for Points-in-Time (POI) channels. + +Exercises the full solve pipeline for ``query.poi_channel(...)`` against the shared +``basic_narrow_db`` (wide) and ``narrow_db`` (EAV) fixtures, which carry POI channels +on ``container_id = 1`` alongside the existing sample channels (see conftest / +``poi_channels.csv``): + +- ``channel_id = 90`` — a **string** DTC-code channel (``P0301`` / ``P0420`` / ``P0301``) +- ``channel_id = 91`` — a **numeric** DTC-count channel (values ``1, 2, 3``) + +Covers: numeric POI unweighted reductions, string POI equality + op gating, the +mix-and-match case (a SAMPLE and a POI channel in one expression), the declared-vs-actual +dtype/series-type assertion, and SAMPLE backward-compatibility. +""" + +import math + +import pytest +import pyspark.sql.types as T +from pyspark.sql import SparkSession + +from impulse_query_engine.analyze.metadata.time_series_expression import PoiValueType +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_query_engine.measurement_db import MeasurementDB + + +class TestNumericPoi: + def test_numeric_poi_mean_is_unweighted(self, spark: SparkSession, basic_narrow_db): + """A numeric POI ``mean()`` is the plain (unweighted) mean of the point values — + POI points have no duration to weight by, unlike ``SampleSeries.mean()``.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") # values 1, 2, 3 + + result = q.select(dtc_count.mean().alias("m")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.m for r in result.collect()} + assert rows[1] == 2.0 # unweighted mean of (1, 2, 3) + + def test_numeric_poi_sum_and_count(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") + + result = q.select( + dtc_count.sum().alias("s"), + dtc_count.count().alias("c"), + ).solve(spark=spark, solver=solver) + + row = {r.container_id: r for r in result.collect()}[1] + assert row.s == 6.0 # 1 + 2 + 3 + assert row.c == 3 + + def test_bare_numeric_poi_selection_types_as_points_in_time( + self, spark: SparkSession, basic_narrow_db + ): + """A bare numeric POI selection serializes as ``array>`` + (PointsInTimeSeries), not the SAMPLE ``binary`` blob type.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count").alias("pit") + + result = q.select(dtc_count).solve(spark=spark, solver=solver) + + assert result.schema["pit"].dataType == T.ArrayType(T.ArrayType(T.DoubleType())) + rows = {r.container_id: r.pit for r in result.collect()} + # three points [t, v], values 1..3 (unweighted, in timestamp order) + assert [pt[1] for pt in rows[1]] == [1.0, 2.0, 3.0] + + +class TestStringPoi: + def test_string_poi_equality_selects_matching_instants( + self, spark: SparkSession, basic_narrow_db + ): + """``string_poi == "P0301"`` yields the instants where the code equals P0301. + + Sampling the count channel at those instants (via ``.where``) picks out the two + P0301 occurrences, proving the string equality drove the point selection. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + # DTC == "P0301" is a PointsInTime; serialize it directly. + result = q.select((dtc == "P0301").alias("hits")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.hits for r in result.collect()} + # P0301 occurs at the 1st and 3rd of the three DTC timestamps. + assert len(rows[1]) == 2 + + def test_string_poi_count_and_sampling_allowed(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(dtc.count().alias("c")).solve(spark=spark, solver=solver) + + assert {r.container_id: r.c for r in result.collect()}[1] == 3 + + @pytest.mark.parametrize("reduction", ["mean", "sum", "min", "max"]) + def test_string_poi_numeric_reduction_rejected_at_build( + self, spark: SparkSession, basic_narrow_db, reduction + ): + """A numeric reduction on a string POI selection is rejected at plan/build time + (before Spark runs), not as a silent NaN.""" + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + selection = getattr(dtc, reduction)().alias("bad") + with pytest.raises(TypeError, match="string-valued"): + q.select(selection)._determine_result_objects_dtypes() + + +class TestMixAndMatch: + """The primary correctness case: a SAMPLE and a POI channel in one expression, both + in the same per-container pandas frame, aligned via ``synchronized``.""" + + def test_sample_channel_sampled_at_poi_instants(self, spark: SparkSession, narrow_db): + """Sample the ``seed`` SAMPLE channel at the instants of the numeric POI channel. + + narrow_db container 1: ``seed`` sample channel has values 1..10 over t=0..10; the + numeric POI channel (91) has points at t = 2, 5, 8. Sampling seed at those instants + picks the seed values valid there. + """ + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc_count = q.poi_channel(channel_name="DTC_count") + + # Sample the sample-series at the POI points (cross-type synchronize). + result = q.select(seed.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.sampled for r in result.collect()} + # three sampled points at the POI instants t = 2, 5, 8 + assert [pt[0] for pt in rows[1]] == [2.0, 5.0, 8.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_and_sample_freeze_frame(self, spark: SparkSession, narrow_db): + """Freeze-frame: sample the seed channel at the instants where DTC == "P0301".""" + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(seed.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at t = 2 and 8 (EAV fixture); seed sampled there. + assert [pt[0] for pt in rows[1]] == [2.0, 8.0] + + def test_sample_channel_sampled_at_poi_instants_wide( + self, spark: SparkSession, basic_narrow_db + ): + """Wide-mode counterpart of the mix-and-match case (``basic_narrow_db``). + + Uses a POI numeric channel to filter/sample a real sample channel. Only + "Ambient Air Temperature" (channel 6) spans all three POI instants in the + ``basic_narrow_csv`` fixture (the other channels end earlier), so it is the + channel sampled here. Proves POI-drives-channel-selection works through the + wide (columns-on-channel_metrics) path, not just the EAV pivot path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants + + result = q.select( + amb.where(dtc_count.to_points_in_time()).alias("sampled") + ).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.sampled for r in result.collect()} + # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv. + poi_instants = [1499929300000000.0, 1499931000000000.0, 1499933000000000.0] + assert [pt[0] for pt in rows[1]] == poi_instants + # Each instant sampled a real Ambient-Air-Temp value (not a miss / NaN). + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db): + """Wide-mode freeze-frame: sample "Ambient Air Temperature" where DTC == "P0301". + + In ``basic_narrow_csv`` P0301 occurs at the 1st and 3rd DTC instants, so the + string-POI equality predicate selects exactly those two instants of the + sample channel — the freeze-frame case resolved via the wide channel path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(amb.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at the 1st and 3rd instants (basic_narrow_csv/poi_channels.csv). + assert [pt[0] for pt in rows[1]] == [1499929300000000.0, 1499933000000000.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + +class TestDeclaredVsActual: + def test_poi_channel_declared_double_on_string_channel_raises( + self, spark: SparkSession, basic_narrow_db + ): + """Declaring ``dtype=double`` on a channel whose silver dtype is ``string`` raises + at solve time — the data stays authoritative.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + # DTC is a numeric-less (string) channel; declaring double resolves rows + # whose value_double is all null → dtype mismatch raised in the solve UDF. + bad = q.poi_channel(channel_name="DTC", dtype=PoiValueType.DOUBLE) + with pytest.raises(Exception, match="dtype mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): + """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. + + The SAMPLE channel's rows carry a real (non-null) validity interval, which + is the signal load_blob validates a POI-declared selector against. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + bad = q.poi_channel(channel_name="Engine RPM") + with pytest.raises(Exception, match="series-type mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + +class TestBackwardCompat: + def test_sample_channel_unaffected_by_poi(self, spark: SparkSession, basic_narrow_db): + """An ordinary SAMPLE ``channel(...)`` selection is unchanged by POI support.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + rpm = q.channel(channel_name="Engine RPM") + + result = q.select(rpm.mean().alias("rpm_mean")).solve(spark=spark, solver=solver) + + rows = {r.container_id for r in result.collect()} + assert rows == {1, 2, 3} diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py index 0e53a802..b236f0e5 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py @@ -611,6 +611,7 @@ def test_col_map_always_returns_internal_names(self, spark): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_mapping_entries_stored_correctly(self, spark): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py index 9ab2b68f..b74b1cd2 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py @@ -505,6 +505,7 @@ def test_col_map_always_returns_internal_names(self, spark: SparkSession): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_config_properties_return_internal_names(self, spark: SparkSession): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py index c692c2ee..524b97db 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py @@ -38,6 +38,7 @@ "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } @@ -148,7 +149,15 @@ class TestColMap: def test_col_map_keys(self, cfg: SolverConfig): """col_map should contain exactly the expected short keys.""" - assert set(cfg.col_map.keys()) == {"cid", "ch", "ts", "te", "val", "conv"} + assert set(cfg.col_map.keys()) == { + "cid", + "ch", + "ts", + "te", + "val", + "conv", + "value_string", + } def test_col_map_default_config(self): """Default SolverConfig col_map should match hardcoded defaults.""" @@ -160,6 +169,7 @@ def test_col_map_default_config(self): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_col_map_consistent_with_properties(self, cfg: SolverConfig): diff --git a/tests/unit/data/basic_narrow_csv/channel_metrics.csv b/tests/unit/data/basic_narrow_csv/channel_metrics.csv index d43f4322..887cb034 100644 --- a/tests/unit/data/basic_narrow_csv/channel_metrics.csv +++ b/tests/unit/data/basic_narrow_csv/channel_metrics.csv @@ -11,3 +11,5 @@ container_id,channel_id,channel_name,group_idx,channel_idx,unit,sample_count,min 2,6,Ambient Air Temperature,2,2,C,57240,21,33,28.793081761006288,1499367269349000,1499372240481000,4971132000,11.514480001738036,DOUBLE 1,7,Vehicle Speed Sensor,3,1,km/h,59625,0,217,68.22906498951782,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE 1,5,Engine RPM,2,1,RPM,59625,0,3658,1490.707790356394,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE +1,90,DTC,0,0,,3,,,,1499929300000000,1499933000000000,3700000,,STRING +1,91,DTC_count,0,0,,3,1,3,2.0,1499929300000000,1499933000000000,3700000,,DOUBLE diff --git a/tests/unit/data/unit_test_csv/1_channel_metrics.csv b/tests/unit/data/unit_test_csv/1_channel_metrics.csv index 46dc9750..59749d53 100644 --- a/tests/unit/data/unit_test_csv/1_channel_metrics.csv +++ b/tests/unit/data/unit_test_csv/1_channel_metrics.csv @@ -1,2 +1,4 @@ container_id,channel_id,value_type,sample_count,nan_ratio,begin_s,end_s,duration_ms,original_sample_count,original_sr,min,max,mean,std,pz1,pz10,pz90,pz99 1,1,numerical,1,1.0,0.0,100.0,1,1,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +1,90,string,3,,2.0,8.0,0,3,,,,,,,,, +1,91,numerical,3,,2.0,8.0,0,3,,1.0,3.0,2.0,,,,, diff --git a/tests/unit/data/unit_test_csv/1_channel_tags.csv b/tests/unit/data/unit_test_csv/1_channel_tags.csv index b5724656..41208776 100644 --- a/tests/unit/data/unit_test_csv/1_channel_tags.csv +++ b/tests/unit/data/unit_test_csv/1_channel_tags.csv @@ -1,2 +1,4 @@ container_id,channel_id,key,value 1,1,seed,0 +1,90,channel_name,DTC +1,91,channel_name,DTC_count From 85b902015f97551f64c822b01a90984cbeda0240 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:07:10 +0200 Subject: [PATCH 06/21] added poi example to notebook Added missing poi in Source Basemodel --- demos/data/reporting/channel_metrics.csv | 38 ++--- demos/reporting_pipeline.ipynb | 144 +++++++++++++++++- .../analyze/query/query_builder.py | 12 +- src/impulse_reporting/config/config_parser.py | 6 +- .../unit/analyze/query/query_builder_test.py | 55 +++++++ .../unit/config/config_parser_test.py | 51 +++++++ 6 files changed, 278 insertions(+), 28 deletions(-) diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 7846174f..46fe67c0 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,19 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME -1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME -2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME -2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME -3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME -3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index edc9c41b..cb9b31c0 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -482,6 +482,51 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3a. Select POI Channels (Diagnostic Trouble Codes)\n", + "\n", + "Not every channel is a continuous signal. A **Points-in-Time (POI)** channel is\n", + "an *event stream*: each value exists **only at its timestamp**, with no validity\n", + "in between. The textbook example is **DTCs** (Diagnostic Trouble Codes) \u2014 the\n", + "fault codes an ECU emits at the instant it detects a problem (`P0301` = cylinder-1\n", + "misfire, \u2026).\n", + "\n", + "POI channels are selected with **`poi_channel(...)`** instead of `channel(...)`.\n", + "Identification is identical (same metadata tags); only the semantics differ \u2014\n", + "a POI channel solves to a `PointsInTimeSeries`, not a `SampleSeries`.\n", + "\n", + "| | `channel(...)` | `poi_channel(...)` |\n", + "|---|---|---|\n", + "| Shape | `[tstart, tend)` intervals | `(t\u1d62, v\u1d62)` points |\n", + "| Valid between points? | yes (interpolated) | **no** |\n", + "| Backed by | `SampleSeries` | `PointsInTimeSeries` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# String POI channel: the DTC code emitted at each fault instant.\n", + "# dtype=\"string\" -> equality is the natural operation (\"when did P0301 occur?\"),\n", + "# never arithmetic or ordering on a code.\n", + "dtc = db.query.poi_channel(\n", + " channel_name=\"DTC\",\n", + " dtype=\"string\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")\n", + "\n", + "# Numeric POI channel: a running fault-occurrence counter.\n", + "dtc_count = db.query.poi_channel(\n", + " channel_name=\"DTC_count\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": { @@ -561,7 +606,8 @@ "\n", "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" ] }, { @@ -608,7 +654,17 @@ " expr=distance_milestones,\n", " desc=\"Each 10 km driven (instant)\",\n", ")\n", - "report.add_event(milestone_event)" + "report.add_event(milestone_event)\n", + "\n", + "# POI freeze-frame: the instants a P0301 misfire was logged.\n", + "# `dtc == \"P0301\"` is a PointsInTime \u2014 the set of timestamps where the\n", + "# string code equals P0301 \u2014 exactly what a PointsInTimeEvent wants.\n", + "p0301_event = PointsInTimeEvent(\n", + " name=\"p0301_misfires\",\n", + " expr=(dtc == \"P0301\"),\n", + " desc=\"Each instant a P0301 misfire code was set\",\n", + ")\n", + "report.add_event(p0301_event)" ] }, { @@ -721,6 +777,19 @@ " event=milestone_event,\n", " desc=\"Speed & RPM at each 10 km milestone\",\n", "))\n", + "\n", + "# \u2500\u2500 POI aggregations \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", + "# Freeze-frame: Engine RPM & Vehicle Speed at each P0301 misfire instant.\n", + "# The POI event supplies the timestamps; the sample channels supply the\n", + "# values valid there \u2014 both series types in one aggregation.\n", + "page.add_aggregation(PointValueAggregator(\n", + " name=\"values_at_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " event=p0301_event,\n", + " desc=\"RPM & Speed at each P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, @@ -754,6 +823,34 @@ "print(\"Calculated channel 'avg_temp' registered\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numeric POI: fault counts per recording\n", + "\n", + "A numeric POI channel reduces like any signal \u2014 but the reductions are\n", + "**unweighted** (points have no duration). `count()` / `max()` on `dtc_count`\n", + "answer \"how many faults did each recording log?\" directly from the query engine.\n", + "\n", + "(The report-level `StatsAggregator` is designed for continuous `SampleSeries`\n", + "inputs, so a per-container POI count is shown here as a direct query instead.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "dtc_summary = db.query.select(\n", + " dtc_count.count().alias(\"n_faults\"),\n", + " dtc_count.max().alias(\"peak_count\"),\n", + ").solve(spark=spark, solver=report.get_solver())\n", + "\n", + "display(dtc_summary.orderBy(\"container_id\"))" + ] + }, { "cell_type": "markdown", "metadata": { @@ -824,7 +921,9 @@ "- **Heatmap** \u2014 RPM vs Speed\n", "- **Table** \u2014 per-container statistics\n", "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", - " (markers only \u2014 values exist only *at* each instant)" + " (markers only \u2014 values exist only *at* each instant)\n", + "\n", + "Includes two **POI** views: Engine RPM sampled at each P0301 misfire (freeze-frame), and fault-code counts per recording." ] }, { @@ -980,7 +1079,42 @@ "ax.set_title(\"Speed & RPM at Each 10 km Milestone\")\n", "ax.legend()\n", "plt.tight_layout()\n", - "plt.show()" + "plt.show()\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 5. POI \u2014 Engine RPM at Each P0301 Misfire (freeze-frame)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# PointValueAggregator writes to the shared stats_aggregator_fact table;\n", + "# select its visual by name, like the milestone scatter above.\n", + "p0301_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'values_at_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"event_instance_id\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "\n", + "if not p0301_df.empty:\n", + " rpm_hits = p0301_df[p0301_df[\"channel_name\"] == \"Engine RPM\"]\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " for cid, grp in rpm_hits.groupby(\"container_id\"):\n", + " ax.scatter(\n", + " grp[\"event_instance_id\"], grp[\"statistic_value\"],\n", + " label=f\"Container {cid}\", s=90, alpha=0.85,\n", + " edgecolors=\"k\", linewidths=0.5,\n", + " )\n", + " ax.set_xlabel(\"P0301 occurrence #\")\n", + " ax.set_ylabel(\"Engine RPM at fault instant\")\n", + " ax.set_title(\"Freeze-frame: Engine RPM at each P0301 misfire\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfires in the demo data.\")\n", + "" ] }, { @@ -1204,4 +1338,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index a97b4653..26469b8b 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -183,9 +183,10 @@ def poi_channel( Parameters ---------- - dtype : PoiValueType, optional + dtype : PoiValueType or str, optional The POI channel's value data type: ``DOUBLE`` (default, numeric) or - ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts + either the enum or its string value (``"double"`` / ``"string"``). This declared type drives plan-time result typing and string-op gating; it is validated against the silver ``poi_channels.dtype`` at solve time (an actual/declared mismatch raises). @@ -197,13 +198,18 @@ def poi_channel( TimeSeriesSelector A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. """ + # Accept a plain string ("string" / "double") as well as the enum, so + # poi_channel(..., dtype="string") behaves identically to the enum form. + value_type = PoiValueType(dtype) expr = None for k, arg in kwargs.items(): if not expr: expr = TagSelector(k) == str(arg) else: expr = expr & (TagSelector(k) == str(arg)) - return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + return TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=value_type + ) def select(self, *args) -> Self: """ diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index 84f49157..634b215c 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -128,6 +128,10 @@ class Source(BaseModel): Full Unity Catalog path to the channel metrics table. channels_uri : str Full Unity Catalog path to the channels data table. + poi_channels_uri : str, optional + Full Unity Catalog path to the Points-in-Time (POI) channel data table. + Required only when the report selects POI channels via ``poi_channel()``; + omit it for sample-only data models. channel_mapping_table : str, optional Full Unity Catalog path to the channel mapping table. Required when using ``channel_with_alias()`` for logical alias resolution. @@ -148,10 +152,10 @@ class Source(BaseModel): container_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channel_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] + poi_channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] | None = None channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None - #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 008b8117..ef361f92 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -4,6 +4,8 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, TimeSeriesSelector, ) from impulse_query_engine.model.series import Intervals @@ -195,3 +197,56 @@ def test_timeseries_selector_dtype_matches_sample_series_dtype(): ts = TimeSeriesSelector(TagSelector("name") == "test") ss = SampleSeries.empty() assert ts.dtype() == ss.dtype() + + +# --------------------------------------------------------------------------- +# QueryBuilder.poi_channel — dtype accepts the enum OR its string value +# --------------------------------------------------------------------------- +class TestPoiChannelDtypeArg: + """``poi_channel(dtype=...)`` must accept both ``PoiValueType`` and the plain + string value (``"double"`` / ``"string"``). A regression guard: a plain + ``dtype="string"`` used to be stored verbatim (a ``str``, not the enum), so the + ``is PoiValueType.STRING`` identity checks silently fell through and a string + POI channel behaved as numeric — blowing up on the first string comparison. + """ + + def test_default_dtype_is_double(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count") + assert sel.series_type is SeriesType.POINTS_IN_TIME + assert sel.value_type is PoiValueType.DOUBLE + + def test_enum_string_dtype(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_dtype_coerced_to_enum(self, narrow_db): + # the design-doc form: poi_channel(..., dtype="string") + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_double_dtype_coerced_to_enum(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count", dtype="double") + assert sel.value_type is PoiValueType.DOUBLE + + def test_invalid_dtype_raises(self, narrow_db): + with pytest.raises(ValueError): + narrow_db.query.poi_channel(channel_name="DTC", dtype="int") + + def test_string_poi_types_as_struct_regardless_of_arg_form(self, narrow_db): + # both arg forms must produce an identical string-typed result dtype + enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + str_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert enum_sel.dtype() == str_sel.dtype() + # string POI serializes as array>, not array> + assert isinstance(str_sel.dtype(), T.ArrayType) + assert isinstance(str_sel.dtype().elementType, T.StructType) + + def test_string_poi_equality_evaluates_to_points_in_time(self, narrow_db): + # dtype="string" must yield a string series so `== "code"` works at plan time + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert (dtc == "P0301").evaluation_type() is PointsInTime + + def test_string_poi_mean_rejected_at_build_time(self, narrow_db): + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + with pytest.raises(TypeError, match="string-valued"): + dtc.mean().evaluation_type() diff --git a/tests/impulse_reporting/unit/config/config_parser_test.py b/tests/impulse_reporting/unit/config/config_parser_test.py index 64189af2..34c99921 100644 --- a/tests/impulse_reporting/unit/config/config_parser_test.py +++ b/tests/impulse_reporting/unit/config/config_parser_test.py @@ -113,6 +113,57 @@ def test_impulse_config_drop_implausible_data_enabled(): assert config.query_engine.drop_implausible_data is True +# --------------------------------------------------------------------------- +# Source.poi_channels_uri — must survive parsing AND reach the MeasurementDB. +# Regression: the field was missing from the Source model, so pydantic silently +# dropped it and the whole reporting-layer POI path was inert (has_poi_channels +# always False) even when the config supplied a poi_channels_uri. +# --------------------------------------------------------------------------- +def test_source_poi_channels_uri_parsed(): + config_json = { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + config = ImpulseConfig.model_validate(config_json) + assert config.source.poi_channels_uri == "impulse_demo.silver.poi_channels" + + +def test_source_poi_channels_uri_defaults_to_none(): + config = ImpulseConfig.model_validate(impulse_config_JSON) + assert config.source.poi_channels_uri is None + + +def test_poi_channels_uri_reaches_measurement_db(): + """End-to-end passthrough: a poi_channels_uri in the config makes the built + MeasurementDB POI-aware (this is what was silently broken).""" + from unittest.mock import create_autospec + + from databricks.sdk import WorkspaceClient + + from impulse_reporting.core.report import Report + + with_poi = ImpulseConfig.model_validate( + { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + ) + db = Report.create_measurement_db(with_poi, create_autospec(WorkspaceClient)) + assert db.has_poi_channels() + assert db.config.poi_channels_uri == "impulse_demo.silver.poi_channels" + + # ...and a config without it stays POI-unaware. + without_poi = ImpulseConfig.model_validate(impulse_config_JSON) + db2 = Report.create_measurement_db(without_poi, create_autospec(WorkspaceClient)) + assert not db2.has_poi_channels() + + def test_impulse_config_drop_implausible_data_rejects_rle(): """drop_implausible_data=True with RLE data must raise ValidationError. From c7572cf9d0342997dd0074909ba02b19f4b402ef Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:27:21 +0200 Subject: [PATCH 07/21] added more poi features to the demo notebook --- demos/reporting_pipeline.ipynb | 77 ++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index cb9b31c0..fee38e69 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -583,7 +583,17 @@ "\n", "# Instant the trip odometer crosses each additional\n", "# 10 km \u2014 a set of points in time, not an interval.\n", - "distance_milestones = (distance_km % 10).falling_edges()" + "distance_milestones = (distance_km % 10).falling_edges()\n", + "\n", + "# POI as a time-window anchor. A POI instant has no duration, but we often\n", + "# want the signal *around* it. `.expand(w)` turns each P0301 instant into a\n", + "# [t - w, t + w] interval (w in the data's time unit \u2014 microseconds here),\n", + "# so `\u00b110 s` is 10e6. Overlapping windows are merged.\n", + "WINDOW_US = 10e6 # \u00b110 seconds\n", + "p0301_window = (dtc == \"P0301\").expand(WINDOW_US)\n", + "\n", + "# All Engine RPM samples recorded within \u00b110 s of a misfire.\n", + "rpm_around_p0301 = eng_rpm.where(p0301_window)" ] }, { @@ -607,7 +617,8 @@ "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)\n", + "- **BasicEvent on a POI window** \u2014 `(dtc == \"P0301\").expand(\u00b110 s)` turns each fault instant into an interval, so you can aggregate the signal *around* each event" ] }, { @@ -664,7 +675,16 @@ " expr=(dtc == \"P0301\"),\n", " desc=\"Each instant a P0301 misfire code was set\",\n", ")\n", - "report.add_event(p0301_event)" + "report.add_event(p0301_event)\n", + "\n", + "# The \u00b110 s misfire windows as an interval event, so aggregations can run\n", + "# \"within 10 s of a P0301\" the same way they run within any other event.\n", + "p0301_window_event = BasicEvent(\n", + " name=\"p0301_window\",\n", + " expr=p0301_window,\n", + " desc=\"Within \u00b110 s of a P0301 misfire\",\n", + ")\n", + "report.add_event(p0301_window_event)" ] }, { @@ -790,6 +810,19 @@ " desc=\"RPM & Speed at each P0301 misfire\",\n", "))\n", "\n", + "\n", + "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", + "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", + "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", + "page.add_aggregation(StatsAggregator(\n", + " name=\"signals_around_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " statistics=[\"min\", \"mean\", \"max\"],\n", + " event=p0301_window_event,\n", + " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, @@ -1114,7 +1147,43 @@ " plt.show()\n", "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", - "" + "\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "win_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'signals_around_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "if not win_df.empty:\n", + " rpm_win = (\n", + " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", + " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", + " values=\"statistic_value\")\n", + " .reset_index()\n", + " )\n", + " fig, ax = plt.subplots(figsize=(9, 4))\n", + " x = range(len(rpm_win))\n", + " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", + " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", + " label=\"min\u2013max range\")\n", + " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", + " ax.set_xticks(list(x))\n", + " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", + " ax.set_ylabel(\"Engine RPM\")\n", + " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfire windows in the demo data.\")" ] }, { From 785b7898c4fc23c85b878fd3b3b0139c3e2df086 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:34:26 +0200 Subject: [PATCH 08/21] switched statsagg to histogramm to showcase poi extension --- demos/reporting_pipeline.ipynb | 62 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index fee38e69..327a4e90 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -811,16 +811,18 @@ "))\n", "\n", "\n", - "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", - "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", - "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", - "page.add_aggregation(StatsAggregator(\n", - " name=\"signals_around_p0301\",\n", - " input_expressions=[eng_rpm, veh_spd],\n", - " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", - " statistics=[\"min\", \"mean\", \"max\"],\n", + "# Derived-value-around-POI: the Engine RPM distribution within \u00b110 s of each\n", + "# misfire. The window event filters the (SampleSeries) channel to those\n", + "# intervals, so a duration-weighted histogram shows *what the engine was doing*\n", + "# around the fault \u2014 richer than a single min/mean/max.\n", + "page.add_aggregation(HistogramDuration(\n", + " name=\"rpm_around_p0301\",\n", + " base_expr=eng_rpm,\n", + " bins=[float(i) for i in range(0, 5000, 250)],\n", " event=p0301_window_event,\n", - " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + " desc=\"Engine RPM distribution within \u00b110 s of a P0301 misfire\",\n", + " channel_name=\"Engine RPM\",\n", + " bins_unit=\"RPM\", values_unit=\"s\",\n", "))\n", "\n", "print(f\"{len(page.aggregations)} aggregations added\")" @@ -1148,38 +1150,30 @@ "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", "\n", - "\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# 6. POI WINDOW \u2014 Engine RPM distribution within \u00b110 s of each P0301\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "win_df = (\n", - " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + "win_hist = (\n", + " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", - " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", - " .filter(\"name = 'signals_around_p0301'\"),\n", + " spark.read.table(f\"{T}_histogram_dimension\")\n", + " .filter(\"name = 'rpm_around_p0301'\"),\n", " on=\"visual_id\",\n", " )\n", - " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .groupBy(\"bin_id\", \"bin_name\")\n", + " .agg(F.sum(\"hist_value\").alias(\"total_us\"))\n", + " .orderBy(\"bin_id\")\n", " .toPandas()\n", ")\n", - "if not win_df.empty:\n", - " rpm_win = (\n", - " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", - " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", - " values=\"statistic_value\")\n", - " .reset_index()\n", - " )\n", - " fig, ax = plt.subplots(figsize=(9, 4))\n", - " x = range(len(rpm_win))\n", - " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", - " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", - " label=\"min\u2013max range\")\n", - " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", - " ax.set_xticks(list(x))\n", - " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", - " ax.set_ylabel(\"Engine RPM\")\n", - " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", - " ax.legend()\n", + "if not win_hist.empty:\n", + " win_hist[\"duration_s\"] = win_hist[\"total_us\"] / 1e6\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " ax.bar(win_hist[\"bin_name\"], win_hist[\"duration_s\"],\n", + " color=\"indianred\", edgecolor=\"white\")\n", + " ax.set_xlabel(\"Engine RPM bin\")\n", + " ax.set_ylabel(\"Duration (s)\")\n", + " ax.set_title(\"Engine RPM distribution within \u00b110 s of a P0301 misfire\")\n", + " plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", " plt.tight_layout()\n", " plt.show()\n", "else:\n", From 8234698f37b8bbcf8bd3552718cffd030c916d39 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 15 Jun 2026 14:40:10 +0200 Subject: [PATCH 09/21] draft solution for fork pr handling --- .github/workflows/acceptance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 96819e29..218c38b9 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,6 +37,10 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write + # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore + # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested + # by the reviewer(s) / maintainer(s) before merging. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -63,6 +67,8 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write + # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 50c41aae4f61ef9d5b03520204018699ddf1f4ff Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:59:22 +0200 Subject: [PATCH 10/21] wip corrected github action --- .github/workflows/acceptance.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 218c38b9..96819e29 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,10 +37,6 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write - # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore - # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested - # by the reviewer(s) / maintainer(s) before merging. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -67,8 +63,6 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write - # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From ebdf05934a9d7f07cc4db42bfafcbf8224149bfc Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 14:18:30 +0200 Subject: [PATCH 11/21] added poi_series_integration.md and marked differences from design to impl --- src/impulse_reporting/config/config_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index 634b215c..fcbd9d55 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -156,6 +156,7 @@ class Source(BaseModel): channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None + #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ From 6ad02f57c8f6981a62df5b473f1a4f99f3beefd2 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:07:10 +0200 Subject: [PATCH 12/21] added poi example to notebook Added missing poi in Source Basemodel --- src/impulse_reporting/config/config_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index fcbd9d55..634b215c 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -156,7 +156,6 @@ class Source(BaseModel): channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None - #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ From 7101b599fbd68865997d78f8e98239c893a8305c Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:56:26 +0200 Subject: [PATCH 13/21] corrected formatting --- .../integration/poi_channel_solve_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py index 8b032ffc..32e194d3 100644 --- a/tests/impulse_query_engine/integration/poi_channel_solve_test.py +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -168,9 +168,9 @@ def test_sample_channel_sampled_at_poi_instants_wide( amb = q.channel(channel_name="Ambient Air Temperature") dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants - result = q.select( - amb.where(dtc_count.to_points_in_time()).alias("sampled") - ).solve(spark=spark, solver=solver) + result = q.select(amb.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) rows = {r.container_id: r.sampled for r in result.collect()} # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv. From 085e72e7a688fbc23d93c2b7257348223c6af280 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 17 Aug 2026 09:11:10 +0200 Subject: [PATCH 14/21] integrated feedback. Moved POI data check to PointsInTimeSeries --- .../metadata/time_series_expression.md | 8 +- .../analyze/query/query_builder.md | 7 +- .../model/series/points_in_time_series.md | 47 ++++++++ .../impulse_reporting/config/config_parser.md | 3 + .../metadata/time_series_expression.py | 12 +- .../analyze/query/query_builder.py | 8 +- .../analyze/query/solvers/default_solver.py | 76 +++---------- .../analyze/query/solvers/series_cache.py | 2 +- .../model/series/points_in_time_series.py | 104 ++++++++++++++++++ .../integration/poi_channel_solve_test.py | 19 ++-- .../unit/analyze/query/query_builder_test.py | 18 +-- .../series/points_in_time_series_test.py | 83 ++++++++++++++ 12 files changed, 292 insertions(+), 95 deletions(-) diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 67c0c089..dcb00893 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -19,10 +19,10 @@ backed by :class:`SampleSeries`. between-point validity, backed by :class:`PointsInTimeSeries`. -## PoiValueType +## SeriesValueType ```python -class PoiValueType(StrEnum) +class SeriesValueType(StrEnum) ``` The value data type of a POI channel — selects its ``poi_channels`` value @@ -48,7 +48,7 @@ class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization) def __init__(expr, uses_alias: bool = False, series_type: SeriesType = SeriesType.SAMPLE, - value_type: PoiValueType = PoiValueType.DOUBLE) + value_type: SeriesValueType = SeriesValueType.DOUBLE) ``` Initialize a TimeSeriesSelector. @@ -64,7 +64,7 @@ unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` identical, only the built object and its result dtype differ. This is the plan-time source of truth for the series type (so ``dtype()`` is correct for a bare POI selection with no per-channel metadata lookup). -- `value_type` (`PoiValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type +- `value_type` (`SeriesValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time typing and string-op gating; validated against the silver ``poi_channels.dtype`` at solve time (assertion contract). diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 1f5fb82e..c2c6c071 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -122,7 +122,7 @@ Create a time series selector for the given channel tags. #### poi\_channel ```python -def poi_channel(dtype: PoiValueType = PoiValueType.DOUBLE, +def poi_channel(dtype: SeriesValueType = SeriesValueType.DOUBLE, **kwargs) -> TimeSeriesSelector ``` @@ -142,8 +142,9 @@ the built object and its result dtype differ. **Arguments**: -- `dtype` (`PoiValueType`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or -``STRING`` (e.g. DTC codes — only sampling and equality apply). This +- `dtype` (`SeriesValueType or str`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or +``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts +either the enum or its string value (``"double"`` / ``"string"``). This declared type drives plan-time result typing and string-op gating; it is validated against the silver ``poi_channels.dtype`` at solve time (an actual/declared mismatch raises). diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index bdf55c39..df772f7d 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -34,6 +34,53 @@ series has no values to infer from and therefore defaults to numeric; use - `tstarts` (`Sized`): Array-like of time points. - `values` (`Sized`): Array-like of values, one per time point. +#### from\_silver + +```python +def from_silver(cls, + tstarts: pd.Series, + values_double: pd.Series, + values_string: pd.Series | None, + value_type: SeriesValueType, + tend: pd.Series | None = None) -> PointsInTimeSeries +``` + +Build a POI series from a resolved silver-layer slice, validating the + +declared ``value_type`` (and, when *tend* is given, the row shape) against +the data the selector actually landed on. + +The selector drives series-type dispatch, but the silver data stays +authoritative: a ``poi_channel(...)`` selector must resolve to genuine POI +rows. This reconciliation lives here — rather than in ``__init__`` — because +it needs information a constructed point series does not carry: **both** +value columns (to tell a mis-declared dtype from a legitimately empty one) +and the ``tend`` column (to detect a SAMPLE channel). ``__init__`` stays a +thin value constructor used throughout the series algebra. + +**Arguments**: + +- `tstarts` (`pandas.Series`): POI timestamps for the resolved rows. +- `values_double` (`pandas.Series`): The numeric value column for the resolved rows. +- `values_string` (`pandas.Series or None`): The string value column, when the frame carries one; ``None`` otherwise. +- `value_type` (`SeriesValueType`): The declared value type. ``STRING`` builds from *values_string*, any +other value builds from *values_double*. +- `tend` (`pandas.Series or None`): The validity-interval end column. A genuine POI row has a null ``tend`` +or a zero-duration interval (``tstart == tend``, should POI ever live in +the SAMPLE ``channels`` table); a real interval (``tend != tstart``) +means the selector resolved to a SAMPLE channel. ``None`` skips the +series-type check. + +**Raises**: + +- `ValueError`: On a series-type mismatch (resolved to a SAMPLE channel) or a +declared-vs-actual dtype mismatch. Fails loudly rather than silently +reading the wrong column (mirrors the unit-conversion conflict check). + +**Returns**: + +`PointsInTimeSeries`: A string-valued series when *value_type* is ``STRING``, else numeric. + #### dtype ```python diff --git a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md index 12099bc9..b6b5456f 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md +++ b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md @@ -82,6 +82,9 @@ configured) regardless of whether ``container_tags_table`` is set. - `container_metrics_table` (`str`): Full Unity Catalog path to the container metrics table. - `channel_metrics_table` (`str`): Full Unity Catalog path to the channel metrics table. - `channels_uri` (`str`): Full Unity Catalog path to the channels data table. +- `poi_channels_uri` (`str`): Full Unity Catalog path to the Points-in-Time (POI) channel data table. +Required only when the report selects POI channels via ``poi_channel()``; +omit it for sample-only data models. - `channel_mapping_table` (`str`): Full Unity Catalog path to the channel mapping table. Required when using ``channel_with_alias()`` for logical alias resolution. - `unit_conversion_table` (`str`): Full Unity Catalog path to the unit conversion table. When set together diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 87bd166a..42a5378e 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -33,7 +33,7 @@ class SeriesType(StrEnum): POINTS_IN_TIME = "POINTS_IN_TIME" -class PoiValueType(StrEnum): +class SeriesValueType(StrEnum): """The value data type of a POI channel — selects its ``poi_channels`` value column and which in-memory :class:`PointsInTimeSeries` variant is built. @@ -656,7 +656,7 @@ def __init__( expr, uses_alias: bool = False, series_type: SeriesType = SeriesType.SAMPLE, - value_type: PoiValueType = PoiValueType.DOUBLE, + value_type: SeriesValueType = SeriesValueType.DOUBLE, ): """ Initialize a TimeSeriesSelector. @@ -675,7 +675,7 @@ def __init__( identical, only the built object and its result dtype differ. This is the plan-time source of truth for the series type (so ``dtype()`` is correct for a bare POI selection with no per-channel metadata lookup). - value_type : PoiValueType, optional + value_type : SeriesValueType, optional For a ``POINTS_IN_TIME`` selection, the declared value data type (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time typing and string-op gating; validated against the silver @@ -696,7 +696,7 @@ def series_type(self) -> SeriesType: return self._series_type @property - def value_type(self) -> PoiValueType: + def value_type(self) -> SeriesValueType: return self._value_type @property @@ -731,7 +731,7 @@ def _empty_points_in_time(self) -> PointsInTimeSeries: and the string-op gating (e.g. ``.mean()`` raising) reflect the declared type before any data is read. """ - if self._value_type is PoiValueType.STRING: + if self._value_type is SeriesValueType.STRING: return PointsInTimeSeries.empty_string() return PointsInTimeSeries.empty() @@ -900,7 +900,7 @@ def from_dict(obj: dict): expr, uses_alias=obj.get("uses_alias", False), series_type=SeriesType(obj.get("series_type", SeriesType.SAMPLE)), - value_type=PoiValueType(obj.get("value_type", PoiValueType.DOUBLE)), + value_type=SeriesValueType(obj.get("value_type", SeriesValueType.DOUBLE)), ) if "alias" in obj and obj["alias"] is not None: m.alias(obj["alias"]) diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 26469b8b..397e1bd7 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -7,7 +7,7 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricSelector from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( - PoiValueType, + SeriesValueType, RequiresDeserialization, SeriesType, TimeSeriesExpression, @@ -164,7 +164,7 @@ def channel_with_alias(self, **kwargs) -> TimeSeriesSelector: return TimeSeriesSelector(expr, uses_alias=True) def poi_channel( - self, dtype: PoiValueType = PoiValueType.DOUBLE, **kwargs + self, dtype: SeriesValueType = SeriesValueType.DOUBLE, **kwargs ) -> TimeSeriesSelector: """ Create a Points-in-Time (POI) channel selector. @@ -183,7 +183,7 @@ def poi_channel( Parameters ---------- - dtype : PoiValueType or str, optional + dtype : SeriesValueType or str, optional The POI channel's value data type: ``DOUBLE`` (default, numeric) or ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts either the enum or its string value (``"double"`` / ``"string"``). This @@ -200,7 +200,7 @@ def poi_channel( """ # Accept a plain string ("string" / "double") as well as the enum, so # poi_channel(..., dtype="string") behaves identically to the enum form. - value_type = PoiValueType(dtype) + value_type = SeriesValueType(dtype) expr = None for k, arg in kwargs.items(): if not expr: diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index e56eb95a..53147b78 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -11,10 +11,7 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression -from impulse_query_engine.analyze.metadata.time_series_expression import ( - PoiValueType, - SeriesType, -) +from impulse_query_engine.analyze.metadata.time_series_expression import SeriesType from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries @@ -140,7 +137,7 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ series_type : SeriesType, optional The calling selector's series type; ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`. ``None`` (default) => SAMPLE. - value_type : PoiValueType, optional + value_type : SeriesValueType, optional For a POI selector, its declared value type; ``STRING`` reads the string value column, otherwise the numeric one. @@ -152,12 +149,20 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ s = self.pdf.iloc[lo:hi] if series_type == SeriesType.POINTS_IN_TIME: - self._assert_poi_data(s, value_type) - if value_type == PoiValueType.STRING: - # value_string is a populated string column, so the constructor - # infers the string value type from it. - return PointsInTimeSeries(s[self._ts_col], s[self._value_string_col]) - return PointsInTimeSeries(s[self._ts_col], s[self._val_col]) + value_string = ( + s[self._value_string_col] + if self._value_string_col is not None and self._value_string_col in s.columns + else None + ) + # from_silver owns the declared-vs-actual reconciliation: it needs both + # value columns and tend, which a bare PointsInTimeSeries does not carry. + return PointsInTimeSeries.from_silver( + s[self._ts_col], + s[self._val_col], + value_string, + value_type, + tend=s[self._te_col], + ) values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: @@ -166,53 +171,6 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) - def _assert_poi_data(self, s, value_type) -> None: - """Validate a POI selector against the data it resolved to. - - The selector drives series-type dispatch, but the silver data stays - authoritative: a ``poi_channel(...)`` selector must land on genuine POI - rows. POI rows carry a null ``tend`` (a point has no validity interval), - whereas SAMPLE rows always carry a real ``tend`` (non-nullable in - ``channels``); so a non-null ``tend`` on a POI-declared slice means the - selector was pointed at a SAMPLE channel. A ``STRING`` declaration - additionally requires a populated ``value_string``. Either mismatch raises - rather than silently reading the wrong column (mirrors the unit-conversion - conflict check). - """ - if len(s) == 0: - return - if pd.notna(s[self._te_col].iloc[0]): - raise ValueError( - "POI channel series-type mismatch: poi_channel(...) resolved to a SAMPLE " - "channel (its rows carry a validity interval). Use channel(...) for SAMPLE " - "channels and poi_channel(...) for POINTS_IN_TIME channels." - ) - - has_string_col = self._value_string_col is not None and self._value_string_col in s.columns - string_all_null = has_string_col and s[self._value_string_col].isna().all() - double_all_null = s[self._val_col].isna().all() - - if value_type == PoiValueType.STRING: - # A string POI channel must carry string values; all-null means the - # channel is actually numeric (declared the wrong dtype). - if not has_string_col or string_all_null: - raise ValueError( - "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a channel " - "with no string values (it is a numeric POI channel). Pass dtype=double to " - "poi_channel(...)." - ) - else: - # A numeric POI channel must carry numeric values; all-null numeric - # with populated string values means the channel is actually a string - # channel (declared the wrong dtype). - if double_all_null and has_string_col and not string_all_null: - raise ValueError( - "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a channel " - "whose numeric values are all null (it is a string POI channel). Pass " - "dtype=string to poi_channel(...)." - ) - - class DefaultSolver(QuerySolver): """ The default query-engine solver. Adapts to the shape of the silver layer. @@ -1146,7 +1104,7 @@ def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), F.lit(None).cast(T.LongType()).alias(cfg.tend_col), F.col(cfg.poi_value_double_col).alias(cfg.value_col), - F.col(cfg.poi_value_string_col).alias(cfg.poi_value_string_col), + F.col(cfg.poi_value_string_col) ) return channels_q.unionByName(poi_proj, allowMissingColumns=True) diff --git a/src/impulse_query_engine/analyze/query/solvers/series_cache.py b/src/impulse_query_engine/analyze/query/solvers/series_cache.py index aed9d810..a64af79c 100644 --- a/src/impulse_query_engine/analyze/query/solvers/series_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/series_cache.py @@ -57,7 +57,7 @@ def load_blob( ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`, otherwise a :class:`SampleSeries`. ``None`` (default) means SAMPLE, so callers that predate POI are unchanged. - value_type : PoiValueType, optional + value_type : SeriesValueType, optional For a ``POINTS_IN_TIME`` selector, its declared value type (``DOUBLE`` / ``STRING``) — selects the numeric vs string value column. Ignored for SAMPLE. The declared type is validated against diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 44d8dcd0..39b125dc 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -4,6 +4,7 @@ import functools from collections.abc import Callable, Sized +from typing import TYPE_CHECKING import numpy as np import numpy.typing as npt @@ -12,6 +13,12 @@ from .intervals import Intervals from .points_in_time import PointsInTime from .sample_series import SampleSeries +import pandas as pd + +if TYPE_CHECKING: + # For annotations only; the runtime use lives inside from_silver as a local + # import to avoid a circular import (time_series_expression imports this module). + from ...analyze.metadata.time_series_expression import SeriesValueType FloatOrNaN = float | np.float64 @@ -77,6 +84,103 @@ def __init__(self, tstarts: Sized, values: Sized): else: self.values = np.array(values, dtype=np.float64) + + @classmethod + def from_silver( + cls, + tstarts: pd.Series, + values_double: pd.Series, + values_string: pd.Series | None, + value_type: SeriesValueType, + tend: pd.Series | None = None, + ) -> PointsInTimeSeries: + """Build a POI series from a resolved silver-layer slice, validating the + declared ``value_type`` (and, when *tend* is given, the row shape) against + the data the selector actually landed on. + + The selector drives series-type dispatch, but the silver data stays + authoritative: a ``poi_channel(...)`` selector must resolve to genuine POI + rows. This reconciliation lives here — rather than in ``__init__`` — because + it needs information a constructed point series does not carry: **both** + value columns (to tell a mis-declared dtype from a legitimately empty one) + and the ``tend`` column (to detect a SAMPLE channel). ``__init__`` stays a + thin value constructor used throughout the series algebra. + + Parameters + ---------- + tstarts : pandas.Series + POI timestamps for the resolved rows. + values_double : pandas.Series + The numeric value column for the resolved rows. + values_string : pandas.Series or None + The string value column, when the frame carries one; ``None`` otherwise. + value_type : SeriesValueType + The declared value type. ``STRING`` builds from *values_string*, any + other value builds from *values_double*. + tend : pandas.Series or None, optional + The validity-interval end column. A genuine POI row has a null ``tend`` + or a zero-duration interval (``tstart == tend``, should POI ever live in + the SAMPLE ``channels`` table); a real interval (``tend != tstart``) + means the selector resolved to a SAMPLE channel. ``None`` skips the + series-type check. + + Returns + ------- + PointsInTimeSeries + A string-valued series when *value_type* is ``STRING``, else numeric. + + Raises + ------ + ValueError + On a series-type mismatch (resolved to a SAMPLE channel) or a + declared-vs-actual dtype mismatch. Fails loudly rather than silently + reading the wrong column (mirrors the unit-conversion conflict check). + """ + # Local runtime import (see the TYPE_CHECKING block above): a module-scope + # import would be circular, as time_series_expression imports this module. + from ...analyze.metadata.time_series_expression import SeriesValueType + + if len(tstarts) > 0: + # Series-type: a genuine POI row has a null tend or a zero-duration + # interval (tstart == tend). Only a real interval (tend != tstart) + # means the selector resolved to a SAMPLE channel. + if tend is not None: + te = tend.iloc[0] + if pd.notna(te) and te != tstarts.iloc[0]: + raise ValueError( + "POI channel series-type mismatch: poi_channel(...) resolved to a " + "SAMPLE channel (its rows carry a validity interval). Use channel(...) " + "for SAMPLE channels and poi_channel(...) for POINTS_IN_TIME channels." + ) + + string_all_null = values_string is not None and values_string.isna().all() + double_all_null = values_double.isna().all() + + if value_type == SeriesValueType.STRING: + # A string POI channel must carry string values; all-null means the + # channel is actually numeric (declared the wrong dtype). + if values_string is None or string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a " + "channel with no string values (it is a numeric POI channel). Pass " + "dtype=double to poi_channel(...)." + ) + elif double_all_null and values_string is not None and not string_all_null: + # A numeric POI channel must carry numeric values; all-null numeric + # with populated string values means the channel is actually string. + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a " + "channel whose numeric values are all null (it is a string POI channel). " + "Pass dtype=string to poi_channel(...)." + ) + + if value_type == SeriesValueType.STRING: + # values_string is the populated string column (validated non-null above + # for a non-empty slice), so the constructor infers the string type from it. + return cls(tstarts, values_string if values_string is not None else []) + return cls(tstarts, values_double) + + def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py index 32e194d3..cad8d2fa 100644 --- a/tests/impulse_query_engine/integration/poi_channel_solve_test.py +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -19,7 +19,7 @@ import pyspark.sql.types as T from pyspark.sql import SparkSession -from impulse_query_engine.analyze.metadata.time_series_expression import PoiValueType +from impulse_query_engine.analyze.metadata.time_series_expression import SeriesValueType from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver from impulse_query_engine.measurement_db import MeasurementDB @@ -79,7 +79,7 @@ def test_string_poi_equality_selects_matching_instants( """ solver = DefaultSolver(spark) q = basic_narrow_db.query - dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) # DTC == "P0301" is a PointsInTime; serialize it directly. result = q.select((dtc == "P0301").alias("hits")).solve(spark=spark, solver=solver) @@ -91,7 +91,7 @@ def test_string_poi_equality_selects_matching_instants( def test_string_poi_count_and_sampling_allowed(self, spark: SparkSession, basic_narrow_db): solver = DefaultSolver(spark) q = basic_narrow_db.query - dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) result = q.select(dtc.count().alias("c")).solve(spark=spark, solver=solver) @@ -104,7 +104,7 @@ def test_string_poi_numeric_reduction_rejected_at_build( """A numeric reduction on a string POI selection is rejected at plan/build time (before Spark runs), not as a silent NaN.""" q = basic_narrow_db.query - dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) selection = getattr(dtc, reduction)().alias("bad") with pytest.raises(TypeError, match="string-valued"): q.select(selection)._determine_result_objects_dtypes() @@ -142,7 +142,7 @@ def test_string_poi_and_sample_freeze_frame(self, spark: SparkSession, narrow_db solver = DefaultSolver(spark) q = narrow_db.query seed = q.channel(seed="0") - dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) result = q.select(seed.where(dtc == "P0301").alias("frozen")).solve( spark=spark, solver=solver @@ -190,7 +190,7 @@ def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db solver = DefaultSolver(spark) q = basic_narrow_db.query amb = q.channel(channel_name="Ambient Air Temperature") - dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) result = q.select(amb.where(dtc == "P0301").alias("frozen")).solve( spark=spark, solver=solver @@ -213,15 +213,16 @@ def test_poi_channel_declared_double_on_string_channel_raises( q = basic_narrow_db.query # DTC is a numeric-less (string) channel; declaring double resolves rows # whose value_double is all null → dtype mismatch raised in the solve UDF. - bad = q.poi_channel(channel_name="DTC", dtype=PoiValueType.DOUBLE) + bad = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.DOUBLE) with pytest.raises(Exception, match="dtype mismatch"): q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. - The SAMPLE channel's rows carry a real (non-null) validity interval, which - is the signal load_blob validates a POI-declared selector against. + The SAMPLE channel's rows carry a real validity interval (``tend != tstart``), + which is the signal ``PointsInTimeSeries.from_silver`` validates a POI-declared + selector against. (A zero-duration ``tstart == tend`` row would be accepted.) """ solver = DefaultSolver(spark) q = basic_narrow_db.query diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index ef361f92..1d74cc00 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -4,7 +4,7 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( - PoiValueType, + SeriesValueType, SeriesType, TimeSeriesSelector, ) @@ -203,30 +203,30 @@ def test_timeseries_selector_dtype_matches_sample_series_dtype(): # QueryBuilder.poi_channel — dtype accepts the enum OR its string value # --------------------------------------------------------------------------- class TestPoiChannelDtypeArg: - """``poi_channel(dtype=...)`` must accept both ``PoiValueType`` and the plain + """``poi_channel(dtype=...)`` must accept both ``SeriesValueType`` and the plain string value (``"double"`` / ``"string"``). A regression guard: a plain ``dtype="string"`` used to be stored verbatim (a ``str``, not the enum), so the - ``is PoiValueType.STRING`` identity checks silently fell through and a string + ``is SeriesValueType.STRING`` identity checks silently fell through and a string POI channel behaved as numeric — blowing up on the first string comparison. """ def test_default_dtype_is_double(self, narrow_db): sel = narrow_db.query.poi_channel(channel_name="DTC_count") assert sel.series_type is SeriesType.POINTS_IN_TIME - assert sel.value_type is PoiValueType.DOUBLE + assert sel.value_type is SeriesValueType.DOUBLE def test_enum_string_dtype(self, narrow_db): - sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) - assert sel.value_type is PoiValueType.STRING + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) + assert sel.value_type is SeriesValueType.STRING def test_plain_string_dtype_coerced_to_enum(self, narrow_db): # the design-doc form: poi_channel(..., dtype="string") sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") - assert sel.value_type is PoiValueType.STRING + assert sel.value_type is SeriesValueType.STRING def test_plain_string_double_dtype_coerced_to_enum(self, narrow_db): sel = narrow_db.query.poi_channel(channel_name="DTC_count", dtype="double") - assert sel.value_type is PoiValueType.DOUBLE + assert sel.value_type is SeriesValueType.DOUBLE def test_invalid_dtype_raises(self, narrow_db): with pytest.raises(ValueError): @@ -234,7 +234,7 @@ def test_invalid_dtype_raises(self, narrow_db): def test_string_poi_types_as_struct_regardless_of_arg_form(self, narrow_db): # both arg forms must produce an identical string-typed result dtype - enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) str_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") assert enum_sel.dtype() == str_sel.dtype() # string POI serializes as array>, not array> diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index 77f8662b..e22ed278 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -3,14 +3,97 @@ # pylint: disable=missing-function-docstring, redefined-outer-name import numpy as np import numpy.testing as nptest +import pandas as pd import pyspark.sql.types as T import pytest +from impulse_query_engine.analyze.metadata.time_series_expression import SeriesValueType from impulse_query_engine.model.series.intervals import Intervals from impulse_query_engine.model.series.points_in_time import PointsInTime from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries +# --- from_silver (silver-slice factory + declared-vs-actual validation) ------------------------- + + +def _dbl(vals): + return pd.Series(vals, dtype="float64") + + +def _str(vals): + return pd.Series(vals, dtype="object") + + +def test_from_silver_numeric(): + pts = PointsInTimeSeries.from_silver( + _dbl([10, 20, 30]), _dbl([1, 2, 3]), _str([None, None, None]), + SeriesValueType.DOUBLE, tend=_dbl([None, None, None]), + ) + assert pts.dtype() == T.ArrayType(T.ArrayType(T.DoubleType())) + nptest.assert_array_equal(pts.values, [1.0, 2.0, 3.0]) + + +def test_from_silver_string(): + pts = PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([None, None]), _str(["P0301", "P0420"]), + SeriesValueType.STRING, tend=_dbl([None, None]), + ) + assert list(pts.values) == ["P0301", "P0420"] + + +def test_from_silver_declared_string_on_numeric_channel_raises(): + # value_string all-null => the channel is really numeric. + with pytest.raises(ValueError, match="dtype mismatch"): + PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([1, 2]), _str([None, None]), + SeriesValueType.STRING, tend=_dbl([None, None]), + ) + + +def test_from_silver_declared_double_on_string_channel_raises(): + # value_double all-null while value_string is populated => really a string channel. + with pytest.raises(ValueError, match="dtype mismatch"): + PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([None, None]), _str(["P0301", "P0420"]), + SeriesValueType.DOUBLE, tend=_dbl([None, None]), + ) + + +def test_from_silver_sample_channel_raises_series_type_mismatch(): + # A real validity interval (tend != tstart) => selector resolved to a SAMPLE channel. + with pytest.raises(ValueError, match="series-type mismatch"): + PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([1, 2]), None, + SeriesValueType.DOUBLE, tend=_dbl([15, 25]), + ) + + +def test_from_silver_zero_duration_interval_is_allowed(): + # tstart == tend is a zero-duration point (POI stored in the SAMPLE table); + # the relaxed check must NOT treat it as a series-type mismatch. + pts = PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([1, 2]), None, + SeriesValueType.DOUBLE, tend=_dbl([10, 20]), + ) + nptest.assert_array_equal(pts.values, [1.0, 2.0]) + + +def test_from_silver_null_tend_is_allowed(): + pts = PointsInTimeSeries.from_silver( + _dbl([10, 20]), _dbl([1, 2]), None, + SeriesValueType.DOUBLE, tend=_dbl([None, None]), + ) + assert len(pts) == 2 + + +def test_from_silver_empty_skips_validation(): + # Nothing resolved => no mismatch can be asserted; builds an empty series. + pts = PointsInTimeSeries.from_silver( + _dbl([]), _dbl([]), None, SeriesValueType.DOUBLE, tend=_dbl([]), + ) + assert len(pts) == 0 + + # --- core --------------------------------------------------------------------------------------- From 40d8dfe3f85e3e80e0db9e9a06ef20d25fc877c5 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 17 Aug 2026 10:10:24 +0200 Subject: [PATCH 15/21] fixed formatting --- .../analyze/query/solvers/default_solver.py | 3 +- .../model/series/points_in_time_series.py | 2 - .../series/points_in_time_series_test.py | 55 ++++++++++++++----- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 53147b78..e13a9e5d 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -171,6 +171,7 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) + class DefaultSolver(QuerySolver): """ The default query-engine solver. Adapts to the shape of the silver layer. @@ -1104,7 +1105,7 @@ def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), F.lit(None).cast(T.LongType()).alias(cfg.tend_col), F.col(cfg.poi_value_double_col).alias(cfg.value_col), - F.col(cfg.poi_value_string_col) + F.col(cfg.poi_value_string_col), ) return channels_q.unionByName(poi_proj, allowMissingColumns=True) diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 39b125dc..542a6c7e 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -84,7 +84,6 @@ def __init__(self, tstarts: Sized, values: Sized): else: self.values = np.array(values, dtype=np.float64) - @classmethod def from_silver( cls, @@ -180,7 +179,6 @@ def from_silver( return cls(tstarts, values_string if values_string is not None else []) return cls(tstarts, values_double) - def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index e22ed278..057dc5cd 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -26,8 +26,11 @@ def _str(vals): def test_from_silver_numeric(): pts = PointsInTimeSeries.from_silver( - _dbl([10, 20, 30]), _dbl([1, 2, 3]), _str([None, None, None]), - SeriesValueType.DOUBLE, tend=_dbl([None, None, None]), + _dbl([10, 20, 30]), + _dbl([1, 2, 3]), + _str([None, None, None]), + SeriesValueType.DOUBLE, + tend=_dbl([None, None, None]), ) assert pts.dtype() == T.ArrayType(T.ArrayType(T.DoubleType())) nptest.assert_array_equal(pts.values, [1.0, 2.0, 3.0]) @@ -35,8 +38,11 @@ def test_from_silver_numeric(): def test_from_silver_string(): pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([None, None]), _str(["P0301", "P0420"]), - SeriesValueType.STRING, tend=_dbl([None, None]), + _dbl([10, 20]), + _dbl([None, None]), + _str(["P0301", "P0420"]), + SeriesValueType.STRING, + tend=_dbl([None, None]), ) assert list(pts.values) == ["P0301", "P0420"] @@ -45,8 +51,11 @@ def test_from_silver_declared_string_on_numeric_channel_raises(): # value_string all-null => the channel is really numeric. with pytest.raises(ValueError, match="dtype mismatch"): PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([1, 2]), _str([None, None]), - SeriesValueType.STRING, tend=_dbl([None, None]), + _dbl([10, 20]), + _dbl([1, 2]), + _str([None, None]), + SeriesValueType.STRING, + tend=_dbl([None, None]), ) @@ -54,8 +63,11 @@ def test_from_silver_declared_double_on_string_channel_raises(): # value_double all-null while value_string is populated => really a string channel. with pytest.raises(ValueError, match="dtype mismatch"): PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([None, None]), _str(["P0301", "P0420"]), - SeriesValueType.DOUBLE, tend=_dbl([None, None]), + _dbl([10, 20]), + _dbl([None, None]), + _str(["P0301", "P0420"]), + SeriesValueType.DOUBLE, + tend=_dbl([None, None]), ) @@ -63,8 +75,11 @@ def test_from_silver_sample_channel_raises_series_type_mismatch(): # A real validity interval (tend != tstart) => selector resolved to a SAMPLE channel. with pytest.raises(ValueError, match="series-type mismatch"): PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([1, 2]), None, - SeriesValueType.DOUBLE, tend=_dbl([15, 25]), + _dbl([10, 20]), + _dbl([1, 2]), + None, + SeriesValueType.DOUBLE, + tend=_dbl([15, 25]), ) @@ -72,16 +87,22 @@ def test_from_silver_zero_duration_interval_is_allowed(): # tstart == tend is a zero-duration point (POI stored in the SAMPLE table); # the relaxed check must NOT treat it as a series-type mismatch. pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([1, 2]), None, - SeriesValueType.DOUBLE, tend=_dbl([10, 20]), + _dbl([10, 20]), + _dbl([1, 2]), + None, + SeriesValueType.DOUBLE, + tend=_dbl([10, 20]), ) nptest.assert_array_equal(pts.values, [1.0, 2.0]) def test_from_silver_null_tend_is_allowed(): pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), _dbl([1, 2]), None, - SeriesValueType.DOUBLE, tend=_dbl([None, None]), + _dbl([10, 20]), + _dbl([1, 2]), + None, + SeriesValueType.DOUBLE, + tend=_dbl([None, None]), ) assert len(pts) == 2 @@ -89,7 +110,11 @@ def test_from_silver_null_tend_is_allowed(): def test_from_silver_empty_skips_validation(): # Nothing resolved => no mismatch can be asserted; builds an empty series. pts = PointsInTimeSeries.from_silver( - _dbl([]), _dbl([]), None, SeriesValueType.DOUBLE, tend=_dbl([]), + _dbl([]), + _dbl([]), + None, + SeriesValueType.DOUBLE, + tend=_dbl([]), ) assert len(pts) == 0 From a4e0bc776fd90102c4baff88fc19f1dfd5504246 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 17 Aug 2026 14:24:21 +0200 Subject: [PATCH 16/21] Updated missing unit tests Added missing csv for testing Removed dtype column from csvs --- demos/data/reporting/poi_channels.csv | 26 ++++----- docs/impulse/docs/config/configuration.md | 1 + .../docs/data_model/silver_layer_schema.md | 38 +++++++++++++ .../analyze/query/solvers/solver_config.md | 9 --- .../references/query_engine/query_solvers.md | 4 +- .../query_engine/tsal/core_data_model.md | 3 + .../query_engine/tsal/defining_expressions.md | 24 ++++++++ skills/impulse-analyze/SKILL.md | 23 +++++++- skills/impulse-config/SKILL.md | 2 + skills/impulse-data-model/SKILL.md | 8 +++ skills/impulse-tsal/SKILL.md | 6 ++ .../metadata/time_series_expression.py | 9 +-- .../analyze/query/solvers/default_solver.py | 38 +++++++++---- .../analyze/query/solvers/solver_config.py | 5 -- src/impulse_query_engine/schema.py | 10 +--- .../unit/analyze/query/query_builder_test.py | 14 +++++ .../solvers/default_solver_poi_gating_test.py | 57 +++++++++++++++++++ .../unit/measurement_db_poi_test.py | 57 +++++++++++++++++++ .../data/basic_narrow_csv/poi_channels.csv | 7 +++ .../data/unit_test_csv/1_poi_channels.csv | 7 +++ 20 files changed, 294 insertions(+), 54 deletions(-) create mode 100644 tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_poi_gating_test.py create mode 100644 tests/impulse_query_engine/unit/measurement_db_poi_test.py create mode 100644 tests/unit/data/basic_narrow_csv/poi_channels.csv create mode 100644 tests/unit/data/unit_test_csv/1_poi_channels.csv diff --git a/demos/data/reporting/poi_channels.csv b/demos/data/reporting/poi_channels.csv index 84357279..b9dbd60b 100644 --- a/demos/data/reporting/poi_channels.csv +++ b/demos/data/reporting/poi_channels.csv @@ -1,13 +1,13 @@ -container_id,channel_id,timestamp,value_double,value_string,dtype -1,90,1519629856439000,,P0301,string -1,90,1519631856439000,,P0301,string -1,90,1519633356439000,,P0135,string -1,91,1519629856439000,1.0,,double -1,91,1519631856439000,2.0,,double -1,91,1519633356439000,3.0,,double -2,90,1519756824107000,,P0420,string -2,90,1519758824107000,,P0128,string -2,91,1519756824107000,1.0,,double -2,91,1519758824107000,2.0,,double -3,90,1519926478375000,,U0100,string -3,91,1519926478375000,1.0,,double +container_id,channel_id,timestamp,value_double,value_string +1,90,1519629856439000,,P0301 +1,90,1519631856439000,,P0301 +1,90,1519633356439000,,P0135 +1,91,1519629856439000,1.0, +1,91,1519631856439000,2.0, +1,91,1519633356439000,3.0, +2,90,1519756824107000,,P0420 +2,90,1519758824107000,,P0128 +2,91,1519756824107000,1.0, +2,91,1519758824107000,2.0, +3,90,1519926478375000,,U0100 +3,91,1519926478375000,1.0, diff --git a/docs/impulse/docs/config/configuration.md b/docs/impulse/docs/config/configuration.md index 8e7cdc55..132b8e14 100644 --- a/docs/impulse/docs/config/configuration.md +++ b/docs/impulse/docs/config/configuration.md @@ -59,6 +59,7 @@ Maps the silver-layer input tables. | `container_metrics_table` | `str` | Yes | Full Unity Catalog path. Container metadata (timestamps, duration). | | `channel_metrics_table` | `str` | Yes | Full Unity Catalog path. Channel-level statistics. | | `channels_uri` | `str` | Yes | Full Unity Catalog path. Time-series sample data. | +| `poi_channels_uri` | `str` | No | Full Unity Catalog path to the Points-in-Time (POI) channel data table. Required only when selecting POI channels via `poi_channel()`; omit for sample-only data models. | | `container_tags_table` | `str` | No | Full Unity Catalog path. Container EAV tags. | | `channel_tags_table` | `str` | No | Full Unity Catalog path. Channel EAV tags. | | `channel_mapping_table` | `str` | No | Full Unity Catalog path. Logical-to-physical channel alias table. Required when using `QueryBuilder.channel_with_alias()`. In reporting mode the resolved alias-to-physical-channel mapping is materialized to the gold-layer [`channel_mapping_resolution_dimension`](../data_model/gold_layer_event_normalized.md#dimension-tables). | diff --git a/docs/impulse/docs/data_model/silver_layer_schema.md b/docs/impulse/docs/data_model/silver_layer_schema.md index 3df66012..723e3471 100644 --- a/docs/impulse/docs/data_model/silver_layer_schema.md +++ b/docs/impulse/docs/data_model/silver_layer_schema.md @@ -410,6 +410,44 @@ during raw→interval conversion (see `query_engine.raw_encoder`). --- +## poi_channels (optional) + +**Optional** — only needed for [Points-in-Time (POI) channels](../references/query_engine/tsal/core_data_model.md#pointsintimeseries) +selected via `QueryBuilder.poi_channel()`. Omit it for sample-only data models. + +Unlike `channels`, a POI channel's value is defined **only at its timestamp** — there is no +`[tstart, tend)` validity interval. **Table membership is the discriminator**: a +`(container_id, channel_id)` whose data lives in `poi_channels` is a POI channel; one in `channels` +is a sample channel. + +A POI value may be numeric or a string (e.g. an ECU Diagnostic Trouble Code). The two typed value +columns cover both, and **exactly one is populated per row** — chosen by the channel's declared +`dtype` at query time (`poi_channel(dtype='double'|'string')`). + +| Column | Type | Nullable | Description | +|----------------|----------|----------|--------------------------------------------------------| +| `container_id` | `long` | No | Parent container identifier. | +| `channel_id` | `int` | No | Channel identifier. | +| `timestamp` | `long` | No | Point timestamp (microseconds). | +| `value_double` | `double` | Yes | Numeric value (populated for a `double` POI channel). | +| `value_string` | `string` | Yes | String value (populated for a `string` POI channel). | + +#### Internal columns referenced by the framework + +Map any silver column to these via +[`solver_config.poi_channels.column_name_mapping`](../config/configuration.md#solver-column-mappings-and-filters) +when your physical column has a different name. + +| Internal name | Referenced by | +|----------------|------------------------------------------------------------------------------| +| `container_id` | Composite key joining points back to their container | +| `channel_id` | Composite key joining points back to their channel | +| `timestamp` | Point timestamp — becomes the point's `tstart` (with a null `tend`) at solve | +| `value_double` | Numeric point value — consumed by the solve UDF | +| `value_string` | String point value — consumed by the solve UDF for a `string` POI channel | + +--- + ## channel_mapping (optional) Alias-resolution table used by `DefaultSolver` when selectors are diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index e16b99cb..e34c236e 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -260,15 +260,6 @@ def poi_value_string_col() -> str Internal column name for the string value on the poi_channels table. -#### poi\_dtype\_col - -```python -def poi_dtype_col() -> str -``` - -Internal column name for the per-row value-dtype discriminator on poi_channels. - - #### tag\_key\_col ```python diff --git a/docs/impulse/docs/references/query_engine/query_solvers.md b/docs/impulse/docs/references/query_engine/query_solvers.md index 5f3ee0d1..25506e66 100644 --- a/docs/impulse/docs/references/query_engine/query_solvers.md +++ b/docs/impulse/docs/references/query_engine/query_solvers.md @@ -126,8 +126,8 @@ from impulse_query_engine.analyze.query.channels.calculated_channel import Calcu from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver cc = CalculatedChannel( - db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, - {"channel_name": "speed_kmh", "data_key": "CALC"}, + db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, + {"channel_name": "speed_kmh", "data_key": "CALC"}, ) df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) # df: [container_id, channel_id, tstart, tend, value, channel_name, data_key] diff --git a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md index 8f755e8b..95263a00 100644 --- a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md +++ b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md @@ -138,6 +138,9 @@ Because there is no validity between points, the operators differ from `SampleSe `SampleSeries`' duration-weighted reducers. - **`synchronized()` / `synchronized_all()`** align this series with a `SampleSeries` or another `PointsInTimeSeries` onto their shared instants, returning value-carrying point series. +- **String-valued POI series** — produced by `poi_channel(dtype='string')` (e.g. DTC fault codes) — + support only equality (`==` / `!=`) and sampling (`synchronized` / `.where`); arithmetic, ordering, + and numeric reductions raise, since they are meaningless for strings. → API: [`PointsInTimeSeries`](../../api/impulse_query_engine/model/series/points_in_time_series.md) diff --git a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md index 70f609ae..b134b919 100644 --- a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md +++ b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md @@ -31,6 +31,30 @@ Each tag passed to `channel(...)` must be resolvable by the solver — either as table is configured — see [Query Solvers](../query_solvers.md#how-defaultsolver-adapts). ::: +### Points-in-Time (POI) channels + +Some channels record values meaningful only **at an instant** — an ECU Diagnostic Trouble Code +(DTC), a discrete event code — with no validity in between. Select these with +`QueryBuilder.poi_channel()` instead of `channel()`. It resolves channels exactly like `channel()` +(same tag filters), but builds a [`PointsInTimeSeries`](core_data_model.md#pointsintimeseries) from +the [`poi_channels`](../../../data_model/silver_layer_schema.md#poi_channels-optional) table rather +than a `SampleSeries` from `channels`. + +```python +# numeric POI channel (default dtype='double') +dtc_count = db.query.poi_channel(channel_name='DTC_count') + +# string POI channel — e.g. fault codes like "P0301" +dtc = db.query.poi_channel(channel_name='DTC', dtype='string') +faults = dtc == 'P0301' # PointsInTime: the instants the code was P0301 +rpm_at_faults = eng_rpm.where(faults) # freeze-frame: RPM at each fault instant +``` + +`dtype` accepts the `SeriesValueType.DOUBLE` / `SeriesValueType.STRING` enum or the plain string +`'double'` / `'string'` (default `'double'`). A **string** POI channel supports only equality +(`==` / `!=`) and sampling (`.where(...)`) — arithmetic, ordering, and numeric reductions raise, as +they are meaningless for string values. + ### Logical aliases via channel mapping For workflows where a stable logical name should resolve to one of many physical channels through a separately diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 47a86d63..91907cad 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -87,6 +87,26 @@ to collect a pandas DataFrame directly: pdf = db.query.select(eng_rpm.mean().alias("rpm_mean")).toPandas(spark, solver=DefaultSolver(spark)) ``` +## Selecting Points-in-Time (POI) channels + +A POI channel carries values defined *only at* their timestamp (no interval). Select one with +`poi_channel(...)` instead of `channel(...)` — identification (tags / `channel_metrics` columns) is +identical; only the built series type differs: + +```python +# numeric POI channel (default dtype="double") +dtc_count = db.query.poi_channel(channel_name="DTC_count") +# string POI channel (e.g. DTC fault codes) +dtc = db.query.poi_channel(channel_name="DTC", dtype="string") +``` + +- `dtype` is `"double"` (default, numeric) or `"string"`, and accepts either the `SeriesValueType` enum + or the plain string. +- **String** POI series support only equality (`== "P0301"` / `!=`) and sampling (`.where(...)`); + arithmetic, ordering and numeric reductions (`sum`/`mean`/`min`/`max`) are rejected at build time. +- The declared `dtype` is validated against the silver data at solve time (a declared-vs-actual mismatch + raises). See `impulse-data-model` for the `poi_channels` table and `impulse-tsal` for the algebra. + ## Choosing the solver `DefaultSolver(spark)` reads your silver layer. Constructor: @@ -122,7 +142,8 @@ Each selected expression is typed by what it evaluates to (see `impulse-tsal` fo | `SampleSeries` | `BinaryType` (pickle+lz4) | deserialized back into an object | | `Intervals` | `ArrayType(ArrayType(DoubleType))` | nested lists `[[tstart, tend], ...]` | | `PointsInTime` | `ArrayType(DoubleType)` | list `[tstart, ...]` | -| `PointsInTimeSeries` | `ArrayType(ArrayType(DoubleType))` | nested lists `[[tstart, value], ...]` | +| `PointsInTimeSeries` (numeric) | `ArrayType(ArrayType(DoubleType))` | nested lists `[[tstart, value], ...]` | +| `PointsInTimeSeries` (string) | `ArrayType(StructType[tstart:double, value:string])` | list of `(tstart, value)` structs | | scalar | `DoubleType` | the value | For scalar-per-container summaries (means, maxima, counts), `select()` the reducer expressions as diff --git a/skills/impulse-config/SKILL.md b/skills/impulse-config/SKILL.md index 006ec019..2297b50a 100644 --- a/skills/impulse-config/SKILL.md +++ b/skills/impulse-config/SKILL.md @@ -24,6 +24,7 @@ config = { "container_metrics_table": "my_catalog.silver.container_metrics", "channel_metrics_table": "my_catalog.silver.channel_metrics", "channels_uri": "my_catalog.silver.channels", + "poi_channels_uri": "my_catalog.silver.poi_channels", # optional "container_tags_table": "my_catalog.silver.container_tags", # optional "channel_tags_table": "my_catalog.silver.channel_tags", # optional }, @@ -54,6 +55,7 @@ Maps the silver-layer input tables. Values are full Unity Catalog paths (`catalo | `container_metrics_table` | Yes | Container metadata (timestamps, duration). | | `channel_metrics_table` | Yes | Per-channel statistics; channel-selection columns in the wide model. | | `channels_uri` | Yes | Time-series sample data. | +| `poi_channels_uri` | No | Points-in-Time (POI) channel data. Required only when selecting POI channels via `poi_channel()`; omit for sample-only models. | | `container_tags_table` | No | Container EAV tags. Required to use `tag_filters`. | | `channel_tags_table` | No | Channel EAV tags. Required to select channels by tag. | | `channel_mapping_table` | No | Logical→physical alias table. Required for `channel_with_alias()`. | diff --git a/skills/impulse-data-model/SKILL.md b/skills/impulse-data-model/SKILL.md index bdee84d8..efc29b9a 100644 --- a/skills/impulse-data-model/SKILL.md +++ b/skills/impulse-data-model/SKILL.md @@ -25,6 +25,7 @@ in `source` (see `impulse-config`). | `container_metrics` | **Yes** | One row per recording — timestamps, duration, channel count, and any container-level columns. | | `channel_metrics` | **Yes** | One row per `(container_id, channel_id)` — per-channel statistics; also holds channel-selection columns (e.g. `channel_name`) in the wide model. | | `channels` | **Yes** | The time-series sample data (RLE or RAW — see below). | +| `poi_channels` | Optional | Points-in-Time (POI) channel data — a value defined *only at* its timestamp (no interval). Add to select POI channels via `poi_channel()`. | | `container_tags` | Optional | EAV `(container_id, key, value)`. Add for tag-based container filtering. | | `channel_tags` | Optional | EAV `(container_id, channel_id, key, value)`. Add for EAV channel selection. | | `channel_mapping` | Optional | Logical→physical channel alias table (enables `channel_with_alias()`). | @@ -46,6 +47,13 @@ in `source` (see `impulse-config`). `drop_implausible_data=True` (requires RAW). - Extra columns on `channels` are ignored — the engine projects down to the columns above before solving, so it is safe to keep additional bookkeeping columns on the table. +- **`poi_channels` holds Points-in-Time (POI) data** — a value defined *only at* its timestamp, with no + validity interval (`tend`), unlike sample `channels`. Schema: `(container_id long, channel_id int, + timestamp long [epoch µs], value_double double nullable, value_string string nullable)`. A channel is + a POI channel **iff** its data lives in `poi_channels` rather than `channels` — table membership *is* + the series-type discriminator; there is no `series_type`/`dtype` column. Exactly one of `value_double` + / `value_string` is populated per row (numeric vs string POI). Select with `query.poi_channel(...)` + (see `impulse-analyze`). - **Tag tables are strict EAV.** `query.channel(channel_name="Engine RPM")` looks up `channel_tags.value` where `key = 'channel_name'`. Without `channel_tags`, channel selectors match columns on `channel_metrics` instead. diff --git a/skills/impulse-tsal/SKILL.md b/skills/impulse-tsal/SKILL.md index 3d88a179..f48587fe 100644 --- a/skills/impulse-tsal/SKILL.md +++ b/skills/impulse-tsal/SKILL.md @@ -174,6 +174,12 @@ Transitions between them: `debounce(d)`, and `filter(d)` (drop windows shorter than `d`). `where(PointsInTime)` drops any instant that falls in a gap where the signal is not valid, so the result may be shorter than the input. +**String-valued `PointsInTimeSeries`** (from `poi_channel(dtype="string")`, e.g. DTC codes) is +restricted: only equality (`== "code"` / `!=` → `PointsInTime`) and sampling via `.where(...)` apply. +Arithmetic (`+ - * /`), ordering (`> >= < <=`), and numeric reductions (`sum`/`mean`/`min`/`max`) raise a +`TypeError` at build time — the `PointsInTimeSeries` reduction/ordering rows above apply to **numeric** +POI series only. + ## Which type does each consumer need - **Events** (`impulse-events`): `BasicEvent` and `SequenceOfEvents` need **`Intervals`**; diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 42a5378e..5d2dfb3a 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -701,12 +701,13 @@ def value_type(self) -> SeriesValueType: @property def selector_id(self) -> int: - # Include series_type so a SAMPLE and a POINTS_IN_TIME selection of the - # same tag expression resolve as distinct channels. SAMPLE keeps the - # historical id (bare ``str(expr)`` hash) for backward compatibility. + # Include series_type (and, for POI, value_type) so a SAMPLE vs a + # POINTS_IN_TIME selection — or a double- vs string-typed POI selection — + # of the same tag expression resolve as distinct channels. SAMPLE keeps + # the historical id (bare ``str(expr)`` hash) for backward compatibility. if self._series_type is SeriesType.SAMPLE: return zlib.crc32(str(self._expr).encode()) - return zlib.crc32(f"{self._series_type}|{self._expr}".encode()) + return zlib.crc32(f"{self._series_type}|{self._expr}|{self._value_type}".encode()) def dtype(self): """ diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index e13a9e5d..1f55fbb2 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -2,7 +2,7 @@ from collections.abc import Iterable from functools import partial -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pandas as pd import pyspark.sql.functions as F @@ -11,7 +11,10 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression -from impulse_query_engine.analyze.metadata.time_series_expression import SeriesType +from impulse_query_engine.analyze.metadata.time_series_expression import ( + SeriesType, + TimeSeriesExpression, +) from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries @@ -1065,13 +1068,9 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra ) # POI channel data is unioned in AFTER RLE encoding above, so its - # zero-duration points are never run-length merged. The inner join to - # channels_df below drops any POI rows whose channel was not selected, so - # unioning whenever a poi_channels table is configured is correct (a - # pure-SAMPLE query simply matches no POI channel_ids). Which object each - # channel builds is decided by the selector (passed to load_blob), not a - # per-row marker — SAMPLE rows just lack value_string. - q = self._union_poi_channel_data(query, q) + # zero-duration points are never run-length merged. + if DefaultSolver._query_contains_poi_selections(query.selections): + q = self._union_poi_channel_data(query, q) joined_df = q.join( F.broadcast(channels_df), @@ -1080,6 +1079,18 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra container_count = channels_df.select(self.config.container_id_col).distinct().count() return q, joined_df, container_count + @staticmethod + def _query_contains_poi_selections(selections: Iterable[Any]) -> bool: + """True when any *leaf* selector resolved by the query is a POINTS_IN_TIME channel. + + Walks the full expression tree via ``collect_selectors``, + used to skip the POI-table union for pure-SAMPLE queries. + """ + return any( + selector.series_type is SeriesType.POINTS_IN_TIME + for selector in TimeSeriesExpression.collect_selectors(selections) + ) + def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: """Union POI channel-data rows into the (already-encoded) channel-data frame. @@ -1099,11 +1110,16 @@ def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: cfg = self.config poi = db.poi_channels(self.spark) poi = self._apply_column_mapping(poi, cfg.poi_channels.column_name_mapping) + + # ensures poi and channel have the same dtype for start and end timestamp + channels_q_t_start_dtype: T.DataType = channels_q.schema[cfg.tstart_col].dataType + channels_q_t_end_dtype: T.DataType = channels_q.schema[cfg.tend_col].dataType + poi_proj = poi.select( F.col(cfg.container_id_col), F.col(cfg.channel_id_col), - F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), - F.lit(None).cast(T.LongType()).alias(cfg.tend_col), + F.col(cfg.poi_timestamp_col).cast(channels_q_t_start_dtype).alias(cfg.tstart_col), + F.lit(None).cast(channels_q_t_end_dtype).alias(cfg.tend_col), F.col(cfg.poi_value_double_col).alias(cfg.value_col), F.col(cfg.poi_value_string_col), ) diff --git a/src/impulse_query_engine/analyze/query/solvers/solver_config.py b/src/impulse_query_engine/analyze/query/solvers/solver_config.py index 0eb11227..55ad4671 100644 --- a/src/impulse_query_engine/analyze/query/solvers/solver_config.py +++ b/src/impulse_query_engine/analyze/query/solvers/solver_config.py @@ -247,11 +247,6 @@ def poi_value_string_col(self) -> str: """Internal column name for the string value on the poi_channels table.""" return "value_string" - @property - def poi_dtype_col(self) -> str: - """Internal column name for the per-row value-dtype discriminator on poi_channels.""" - return "dtype" - @property def tag_key_col(self) -> str: """Internal column name for the attribute key on the container_tags (EAV) table.""" diff --git a/src/impulse_query_engine/schema.py b/src/impulse_query_engine/schema.py index f64465bc..a0c118b6 100644 --- a/src/impulse_query_engine/schema.py +++ b/src/impulse_query_engine/schema.py @@ -53,14 +53,7 @@ ) # Points-in-Time (POI) channel samples: a value defined only *at* its timestamp -# (no derived tend / validity interval). Two typed value columns plus a per-row -# dtype discriminator, since a POI value may be numeric or a string; exactly one of -# value_double / value_string is populated per row, selected by dtype. -# -# A channel is a POI channel iff its data lives here rather than in ``channels`` — -# table membership *is* the series-type discriminator, so no ``series_type`` column -# is needed on ``channel_metrics``. A given (container_id, channel_id) lives in -# exactly one of ``channels`` / ``poi_channels``. +# (no derived tend / validity interval). POI_CHANNELS_SCHEMA = T.StructType( [ T.StructField("container_id", T.LongType(), nullable=False), @@ -68,7 +61,6 @@ T.StructField("timestamp", T.LongType(), nullable=False), T.StructField("value_double", T.DoubleType()), T.StructField("value_string", T.StringType()), - T.StructField("dtype", T.StringType(), nullable=False), ] ) diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 1d74cc00..92db4312 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -250,3 +250,17 @@ def test_string_poi_mean_rejected_at_build_time(self, narrow_db): dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") with pytest.raises(TypeError, match="string-valued"): dtc.mean().evaluation_type() + + def test_poi_selector_id_distinguishes_value_type(self): + # A double- vs string-typed POI selection of the *same* expression must resolve + # as distinct channels, so their selector_id must differ. + expr = TagSelector("channel_name") == "DTC" + dbl = TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=SeriesValueType.DOUBLE + ) + strg = TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=SeriesValueType.STRING + ) + assert dbl.selector_id != strg.selector_id + # SAMPLE keeps the historical id (independent of value_type), distinct from POI. + assert TimeSeriesSelector(expr).selector_id != dbl.selector_id diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_poi_gating_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_poi_gating_test.py new file mode 100644 index 00000000..750e43fd --- /dev/null +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_poi_gating_test.py @@ -0,0 +1,57 @@ +"""Unit tests for ``DefaultSolver._queryDefaultSolver._query_contains_poi_selections_selections`` (the POI-union gate). +These are Spark-free: they build selectors directly and call the static gate. +""" + +# pylint: disable=missing-function-docstring +from impulse_query_engine.analyze.metadata.tag_expression import TagSelector +from impulse_query_engine.analyze.metadata.time_series_expression import ( + SeriesType, + SeriesValueType, + TimeSeriesSelector, +) +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver + + +def _poi(channel="DTC_count", value_type=SeriesValueType.DOUBLE): + return TimeSeriesSelector( + TagSelector("channel_name") == channel, + series_type=SeriesType.POINTS_IN_TIME, + value_type=value_type, + ) + + +def _sample(name="seed"): + return TimeSeriesSelector(TagSelector(name) == "0") + + +def test_bare_poi_selection_detected(): + assert DefaultSolver._query_contains_poi_selections([_poi()]) is True + + +def test_poi_wrapped_in_aggregation_detected(): + # The regression case: .sum()/.count() wrap the POI selector in a TimeSeriesOp. + assert DefaultSolver._query_contains_poi_selections([_poi().sum()]) is True + assert DefaultSolver._query_contains_poi_selections([_poi().count()]) is True + + +def test_string_poi_equality_op_detected(): + poi = _poi(channel="DTC", value_type=SeriesValueType.STRING) + assert DefaultSolver._query_contains_poi_selections([poi == "P0301"]) is True + + +def test_mixed_sample_and_wrapped_poi_detected(): + assert DefaultSolver._query_contains_poi_selections([_sample().sum(), _poi().sum()]) is True + + +def test_sample_only_not_detected(): + assert DefaultSolver._query_contains_poi_selections([_sample()]) is False + assert DefaultSolver._query_contains_poi_selections([_sample().sum()]) is False + + +def test_empty_selections_not_detected(): + assert DefaultSolver._query_contains_poi_selections([]) is False + + +def test_non_expression_entries_ignored(): + # collect_selectors silently skips non-TimeSeriesExpression items. + assert DefaultSolver._query_contains_poi_selections(["not-an-expression", 42]) is False diff --git a/tests/impulse_query_engine/unit/measurement_db_poi_test.py b/tests/impulse_query_engine/unit/measurement_db_poi_test.py new file mode 100644 index 00000000..8663cb79 --- /dev/null +++ b/tests/impulse_query_engine/unit/measurement_db_poi_test.py @@ -0,0 +1,57 @@ +# pylint: disable=missing-function-docstring +"""Unit tests for POI wiring on MeasurementDBConfig / MeasurementDB. + +Covers the ``poi_channels_uri`` slot on the config factories and the +``has_poi_channels`` / ``poi_channels`` guard — the config-level half of POI +support that needs no SparkSession. +""" + +from unittest.mock import create_autospec + +import pytest +from databricks.sdk import WorkspaceClient + +from impulse_query_engine.measurement_db import MeasurementDB, MeasurementDBConfig + + +def _db(cfg: MeasurementDBConfig) -> MeasurementDB: + return MeasurementDB(cfg, ws=create_autospec(WorkspaceClient)) + + +class TestForDebug: + def test_poi_channels_wired_when_present(self): + cfg = MeasurementDBConfig.for_debug({"channels": object(), "poi_channels": object()}) + assert cfg.poi_channels_uri == "poi_channels" + assert _db(cfg).has_poi_channels() + + def test_poi_channels_none_when_absent(self): + cfg = MeasurementDBConfig.for_debug({"channels": object()}) + assert cfg.poi_channels_uri is None + assert not _db(cfg).has_poi_channels() + + +class TestForUnityCatalog: + def test_poi_channels_uri_wired(self): + cfg = MeasurementDBConfig.for_unity_catalog( + "cat", poi_channels_uri="cat.core.poi_channels" + ) + assert cfg.poi_channels_uri == "cat.core.poi_channels" + assert _db(cfg).has_poi_channels() + + def test_poi_channels_uri_defaults_none(self): + cfg = MeasurementDBConfig.for_unity_catalog("cat") + assert cfg.poi_channels_uri is None + assert not _db(cfg).has_poi_channels() + + +class TestPoiChannelsReaderGuard: + def test_reader_raises_when_not_configured(self): + db = _db(MeasurementDBConfig.for_debug({"channels": object()})) + with pytest.raises(ValueError, match="poi_channels_uri is not configured"): + db.poi_channels(spark=None) + + def test_reader_returns_debug_table(self): + sentinel = object() + db = _db(MeasurementDBConfig.for_debug({"poi_channels": sentinel})) + # debug mode returns the in-memory table object as-is + assert db.poi_channels(spark=None) is sentinel diff --git a/tests/unit/data/basic_narrow_csv/poi_channels.csv b/tests/unit/data/basic_narrow_csv/poi_channels.csv new file mode 100644 index 00000000..7a2be471 --- /dev/null +++ b/tests/unit/data/basic_narrow_csv/poi_channels.csv @@ -0,0 +1,7 @@ +container_id,channel_id,timestamp,value_double,value_string +1,90,1499929300000000,,P0301 +1,90,1499931000000000,,P0420 +1,90,1499933000000000,,P0301 +1,91,1499929300000000,1.0, +1,91,1499931000000000,2.0, +1,91,1499933000000000,3.0, diff --git a/tests/unit/data/unit_test_csv/1_poi_channels.csv b/tests/unit/data/unit_test_csv/1_poi_channels.csv new file mode 100644 index 00000000..4dc692e6 --- /dev/null +++ b/tests/unit/data/unit_test_csv/1_poi_channels.csv @@ -0,0 +1,7 @@ +container_id,channel_id,timestamp,value_double,value_string +1,90,2,,P0301 +1,90,5,,P0420 +1,90,8,,P0301 +1,91,2,1.0, +1,91,5,2.0, +1,91,8,3.0, From 71e5d3d4454ad31326c2fe97340dc89fc423f335 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 19 Aug 2026 14:44:41 +0200 Subject: [PATCH 17/21] Added missing return types Added value_type to PointsInTimeSerie.empty() so it supports string and double in one place --- .../metadata/time_series_expression.py | 57 ++-- .../analyze/query/solvers/default_solver.py | 25 +- .../model/series/points_in_time_series.py | 287 ++++++++---------- .../model/series/sample_series.py | 6 + .../integration/poi_channel_solve_test.py | 34 +-- .../unit/analyze/query/query_builder_test.py | 2 +- .../series/points_in_time_series_test.py | 150 +++------ 7 files changed, 198 insertions(+), 363 deletions(-) diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 5d2dfb3a..223cc4db 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -13,6 +13,7 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagExpression from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries +from impulse_query_engine.model.series.value_type import SeriesValueType if TYPE_CHECKING: from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache @@ -21,33 +22,20 @@ class SeriesType(StrEnum): """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). - ``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is - *valid* (reconstructed by an interpolation method, zero-order hold today), - backed by :class:`SampleSeries`. + ``SAMPLE`` — the default; the time series is considered *valid* within each + ``[tstart_i, tend_i)`` interval: value ``v_i`` was measured at ``tstart_i`` and no + other value was measured until ``tend_i`` (reconstructed by an interpolation method, + zero-order hold today), backed by :class:`SampleSeries`. - ``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no - between-point validity, backed by :class:`PointsInTimeSeries`. + ``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their + timestamps ``(tᵢ, vᵢ)`` with no between-point validity, backed by + :class:`PointsInTimeSeries`. """ SAMPLE = "SAMPLE" POINTS_IN_TIME = "POINTS_IN_TIME" -class SeriesValueType(StrEnum): - """The value data type of a POI channel — selects its ``poi_channels`` value - column and which in-memory :class:`PointsInTimeSeries` variant is built. - - ``DOUBLE`` — numeric points (``poi_channels.value_double``); the full - arithmetic / ordering / reduction operator set applies. - - ``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); - only sampling and equality apply (see :class:`PointsInTimeSeries`). - """ - - DOUBLE = "double" - STRING = "string" - - class RequiresDeserialization: pass @@ -82,7 +70,7 @@ def requires_sync(self): """ return not self.is_single_signal - def dtype(self): + def dtype(self) -> T.DataType: """ Get the default Spark data type. @@ -709,7 +697,7 @@ def selector_id(self) -> int: return zlib.crc32(str(self._expr).encode()) return zlib.crc32(f"{self._series_type}|{self._expr}|{self._value_type}".encode()) - def dtype(self): + def dtype(self) -> T.DataType: """ Returns the Spark data type. @@ -722,20 +710,9 @@ def dtype(self): ``array>`` for string). """ if self._series_type is SeriesType.POINTS_IN_TIME: - return self._empty_points_in_time().dtype() + return PointsInTimeSeries.empty(value_type=self._value_type).dtype() return T.BinaryType() - def _empty_points_in_time(self) -> PointsInTimeSeries: - """Empty POI series carrying this selector's declared value type. - - A string selector must build a string-typed empty series so ``dtype()`` - and the string-op gating (e.g. ``.mean()`` raising) reflect the declared - type before any data is read. - """ - if self._value_type is SeriesValueType.STRING: - return PointsInTimeSeries.empty_string() - return PointsInTimeSeries.empty() - def deserialize(self, d): """ Deserialize a SAMPLE result after collection/toPandas. @@ -758,7 +735,7 @@ def deserialize(self, d): return d return SampleSeries.deserialize(d) - def build(self, cache: SeriesCache): + def build(self, cache: SeriesCache) -> SampleSeries | PointsInTimeSeries: """ Instantiate the selected series from cache data. @@ -784,7 +761,7 @@ def build(self, cache: SeriesCache): candidates = cache.resolve(self) if len(candidates) == 0: if self._series_type is SeriesType.POINTS_IN_TIME: - return self._empty_points_in_time() + return PointsInTimeSeries.empty(value_type=self._value_type) return SampleSeries.empty() # TODO: select candidate mid = candidates.container_id.iloc[0] @@ -921,7 +898,7 @@ def __init__(self, *aliases): self._aliases = aliases TimeSeriesExpression.__init__(self, is_single_signal=True) - def dtype(self): + def dtype(self) -> T.DataType: """ Returns the Spark data type. @@ -932,7 +909,7 @@ def dtype(self): """ return T.BinaryType() - def build(self, cache: SeriesCache) -> SampleSeries: + def build(self, cache: SeriesCache) -> SampleSeries | PointsInTimeSeries: """ Build the time series from cache. @@ -943,8 +920,8 @@ def build(self, cache: SeriesCache) -> SampleSeries: Returns ------- - SampleSeries - Built sample series. + SampleSeries or PointsInTimeSeries + Built series (a ``SampleSeries`` for the SAMPLE-only aliases used today). """ candidates = [alias.build(cache) for alias in self._aliases] # TODO: propery select best candidate diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 1f55fbb2..143144c5 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -13,6 +13,7 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagExpression from impulse_query_engine.analyze.metadata.time_series_expression import ( SeriesType, + SeriesValueType, TimeSeriesExpression, ) from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries @@ -152,20 +153,16 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ s = self.pdf.iloc[lo:hi] if series_type == SeriesType.POINTS_IN_TIME: - value_string = ( - s[self._value_string_col] - if self._value_string_col is not None and self._value_string_col in s.columns - else None - ) - # from_silver owns the declared-vs-actual reconciliation: it needs both - # value columns and tend, which a bare PointsInTimeSeries does not carry. - return PointsInTimeSeries.from_silver( - s[self._ts_col], - s[self._val_col], - value_string, - value_type, - tend=s[self._te_col], - ) + # The selector's declared value type picks the column; the constructor + # reconciles values-vs-type. A channel may carry both value columns — + # we simply read the declared one. + vt = value_type if value_type is not None else SeriesValueType.DOUBLE + value_col = self._value_string_col if vt is SeriesValueType.STRING else self._val_col # todo extend so we have this in config what the col is + if value_col is None or value_col not in s.columns: + raise ValueError( + f"POI channel declared {vt} but its value column is not available" + ) + return PointsInTimeSeries(s[self._ts_col], s[value_col], value_type=vt) values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 542a6c7e..fe4fd2aa 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -13,12 +13,7 @@ from .intervals import Intervals from .points_in_time import PointsInTime from .sample_series import SampleSeries -import pandas as pd - -if TYPE_CHECKING: - # For annotations only; the runtime use lives inside from_silver as a local - # import to avoid a circular import (time_series_expression imports this module). - from ...analyze.metadata.time_series_expression import SeriesValueType +from .value_type import SeriesValueType FloatOrNaN = float | np.float64 @@ -27,10 +22,10 @@ def _numeric_only(method: Callable) -> Callable: """Decorator that rejects the wrapped method on a string-valued series. String-valued :class:`PointsInTimeSeries` support only sampling and equality - (``==`` / ``!=``); arithmetic, ordering and numeric reductions have no meaning - for them. Numpy would either raise (``-``, ``/``, ``mean``) or — worse — - silently succeed with a nonsensical result (``+`` concatenates, ``*`` repeats, - ``sum`` concatenates), so guard those methods explicitly and fail loudly. + (``==`` / ``!=``); arithmetic, ordering and numeric reductions are **not + implemented** for them. Numpy would either raise (``-``, ``/``, ``mean``) or — + worse — silently succeed with a nonsensical result (``+`` concatenates, ``*`` + repeats, ``sum`` concatenates), so guard those methods explicitly and fail loudly. Applied to arithmetic, ordering-comparison and reduction methods. @@ -42,9 +37,9 @@ def _numeric_only(method: Callable) -> Callable: @functools.wraps(method) def wrapper(self: PointsInTimeSeries, *args, **kwargs): - if self._is_string: + if not self._value_type.is_numeric: raise TypeError( - f"{method.__name__} is not supported for string-valued PointsInTimeSeries" + f"{method.__name__} is not supported for non-numeric PointsInTimeSeries" ) return method(self, *args, **kwargs) @@ -52,7 +47,9 @@ def wrapper(self: PointsInTimeSeries, *args, **kwargs): class PointsInTimeSeries: - def __init__(self, tstarts: Sized, values: Sized): + def __init__( + self, tstarts: Sized, values: Sized, value_type: SeriesValueType = SeriesValueType.DOUBLE + ): """ Initialize the PointsInTimeSeries object. @@ -60,126 +57,57 @@ def __init__(self, tstarts: Sized, values: Sized): a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. - The value type (numeric vs string) is inferred from *values*. An **empty** - series has no values to infer from and therefore defaults to numeric; use - :meth:`empty_string` when an explicitly string-typed empty series is needed - (e.g. plan-time result typing of a bare string-POI selection). - Parameters ---------- tstarts : Sized Array-like of time points. values : Sized Array-like of values, one per time point. + value_type : SeriesValueType, optional + The value data type (default ``DOUBLE``). Validated against *values* for a + non-empty series; carried explicitly so an **empty** series stays typed + correctly (there are no values to infer from). ``STRING`` series support + only sampling and equality — see the ``@_numeric_only`` methods. """ - assert len(tstarts) == len(values) - # Timestamps are always numeric. Values may be numeric or string: - # string-valued series support sampling (``synchronized`` / ``.where``) - # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering - # and numeric reductions are rejected (see the ``@_numeric_only`` methods). + self._validate_provided_values(tstarts, values, value_type) self.tstarts = np.array(tstarts, dtype=np.float64) - self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") - if self._is_string: - self.values = np.asarray(values, dtype=object) - else: - self.values = np.array(values, dtype=np.float64) - - @classmethod - def from_silver( - cls, - tstarts: pd.Series, - values_double: pd.Series, - values_string: pd.Series | None, - value_type: SeriesValueType, - tend: pd.Series | None = None, - ) -> PointsInTimeSeries: - """Build a POI series from a resolved silver-layer slice, validating the - declared ``value_type`` (and, when *tend* is given, the row shape) against - the data the selector actually landed on. - - The selector drives series-type dispatch, but the silver data stays - authoritative: a ``poi_channel(...)`` selector must resolve to genuine POI - rows. This reconciliation lives here — rather than in ``__init__`` — because - it needs information a constructed point series does not carry: **both** - value columns (to tell a mis-declared dtype from a legitimately empty one) - and the ``tend`` column (to detect a SAMPLE channel). ``__init__`` stays a - thin value constructor used throughout the series algebra. - - Parameters - ---------- - tstarts : pandas.Series - POI timestamps for the resolved rows. - values_double : pandas.Series - The numeric value column for the resolved rows. - values_string : pandas.Series or None - The string value column, when the frame carries one; ``None`` otherwise. - value_type : SeriesValueType - The declared value type. ``STRING`` builds from *values_string*, any - other value builds from *values_double*. - tend : pandas.Series or None, optional - The validity-interval end column. A genuine POI row has a null ``tend`` - or a zero-duration interval (``tstart == tend``, should POI ever live in - the SAMPLE ``channels`` table); a real interval (``tend != tstart``) - means the selector resolved to a SAMPLE channel. ``None`` skips the - series-type check. - - Returns - ------- - PointsInTimeSeries - A string-valued series when *value_type* is ``STRING``, else numeric. + self._value_type = value_type + self._set_values(value_type, values) + + def _set_values(self, value_type: SeriesValueType, values: Sized) -> None: + # Initializes instance value attribute with the correct dtype + match value_type: + case SeriesValueType.STRING: + self.values = np.asarray(values, dtype=object) + case SeriesValueType.DOUBLE: + self.values = np.array(values, dtype=np.float64) + case _: + raise ValueError(f"Unsupported value_type: {value_type}") - Raises - ------ - ValueError - On a series-type mismatch (resolved to a SAMPLE channel) or a - declared-vs-actual dtype mismatch. Fails loudly rather than silently - reading the wrong column (mirrors the unit-conversion conflict check). - """ - # Local runtime import (see the TYPE_CHECKING block above): a module-scope - # import would be circular, as time_series_expression imports this module. - from ...analyze.metadata.time_series_expression import SeriesValueType - - if len(tstarts) > 0: - # Series-type: a genuine POI row has a null tend or a zero-duration - # interval (tstart == tend). Only a real interval (tend != tstart) - # means the selector resolved to a SAMPLE channel. - if tend is not None: - te = tend.iloc[0] - if pd.notna(te) and te != tstarts.iloc[0]: - raise ValueError( - "POI channel series-type mismatch: poi_channel(...) resolved to a " - "SAMPLE channel (its rows carry a validity interval). Use channel(...) " - "for SAMPLE channels and poi_channel(...) for POINTS_IN_TIME channels." - ) + @staticmethod + def _validate_provided_values( + tstarts: Sized, values: Sized, value_type: SeriesValueType + ) -> None: + # Quick validity checks against the provided args + assert len(tstarts) == len(values) + if len(values) > 0: + # if the provided values are of type string the value_type needs to be SeriesValueType.STRING + is_str = np.asarray(values).dtype.kind in ("U", "S", "O") + assert ( + value_type is SeriesValueType.STRING + ) == is_str, f"Values do not match declared value_type {value_type}" - string_all_null = values_string is not None and values_string.isna().all() - double_all_null = values_double.isna().all() - - if value_type == SeriesValueType.STRING: - # A string POI channel must carry string values; all-null means the - # channel is actually numeric (declared the wrong dtype). - if values_string is None or string_all_null: - raise ValueError( - "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a " - "channel with no string values (it is a numeric POI channel). Pass " - "dtype=double to poi_channel(...)." - ) - elif double_all_null and values_string is not None and not string_all_null: - # A numeric POI channel must carry numeric values; all-null numeric - # with populated string values means the channel is actually string. - raise ValueError( - "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a " - "channel whose numeric values are all null (it is a string POI channel). " - "Pass dtype=string to poi_channel(...)." - ) + @property + def value_type(self) -> SeriesValueType: + """This series' value data type (numeric ``DOUBLE`` or ``STRING``).""" + return self._value_type - if value_type == SeriesValueType.STRING: - # values_string is the populated string column (validated non-null above - # for a non-empty slice), so the constructor infers the string type from it. - return cls(tstarts, values_string if values_string is not None else []) - return cls(tstarts, values_double) + @property + def is_string(self) -> bool: + """Whether this series carries string (rather than numeric) values.""" + return self._value_type is SeriesValueType.STRING - def dtype(self): + def dtype(self) -> T.DataType: """ Returns the Spark data type for PointsInTimeSeries. @@ -193,16 +121,20 @@ def dtype(self): pyspark.sql.types.ArrayType Spark ArrayType matching ``get_data``'s shape for this series' value type. """ - if self._is_string: - return T.ArrayType( - T.StructType( - [ - T.StructField("tstart", T.DoubleType()), - T.StructField("value", T.StringType()), - ] + match self._value_type: + case SeriesValueType.DOUBLE: + return T.ArrayType(T.ArrayType(T.DoubleType())) + case SeriesValueType.STRING: + return T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) ) - ) - return T.ArrayType(T.ArrayType(T.DoubleType())) + case _: + raise ValueError(f"Unsupported value_type: {self._value_type}") def get_data(self) -> list: """ @@ -220,7 +152,7 @@ def get_data(self) -> list: """ if len(self) == 0: return [] - if self._is_string: + if self.is_string: return [[float(t), str(v)] for t, v in zip(self.tstarts, self.values, strict=True)] return np.column_stack([self.tstarts, self.values]).tolist() @@ -430,8 +362,12 @@ def synchronized( pairs = PointsInTimeSeries.plane_sweep(self, other) tstarts = PointsInTimeSeries.__gather(self.tstarts, pairs, 0) return ( - PointsInTimeSeries(tstarts, PointsInTimeSeries.__gather(self.values, pairs, 0)), - PointsInTimeSeries(tstarts, PointsInTimeSeries.__gather(other.values, pairs, 1)), + PointsInTimeSeries( + tstarts, PointsInTimeSeries.__gather(self.values, pairs, 0), self.value_type + ), + PointsInTimeSeries( + tstarts, PointsInTimeSeries.__gather(other.values, pairs, 1), other.value_type + ), ) def synchronized_all( @@ -468,16 +404,39 @@ def synchronized_all( pairs = PointsInTimeSeries.plane_sweep(grid, other) tstarts = PointsInTimeSeries.__gather(grid.tstarts, pairs, 0) new_synced = [ - PointsInTimeSeries(tstarts, PointsInTimeSeries.__gather(s.values, pairs, 0)) + PointsInTimeSeries( + tstarts, PointsInTimeSeries.__gather(s.values, pairs, 0), s.value_type + ) for s in synced_list ] new_synced.append( - PointsInTimeSeries(tstarts, PointsInTimeSeries.__gather(other.values, pairs, 1)) + PointsInTimeSeries( + tstarts, + PointsInTimeSeries.__gather(other.values, pairs, 1), + other.value_type, + ) ) synced_list = new_synced return tuple(synced_list) - def _apply_basic_op(self, operation, other: float | SampleSeries | PointsInTimeSeries): + @staticmethod + def _derive_child_value_type( + operation, # noqa: ARG004 + a: SeriesValueType, # noqa: ARG004 + b: SeriesValueType, # noqa: ARG004 + ) -> SeriesValueType: + """Value type of the child series produced by applying *operation* to operands + whose value types are *a* and *b*. + Currently only op's targeting numeric values are supported, so all resulting types are SeriesValueType.DOUBLE + """ + match operation: + # extend in the future if more SeriesValueType are supported + case _: + return SeriesValueType.DOUBLE + + def _apply_basic_op( + self, operation, other: float | SampleSeries | PointsInTimeSeries + ) -> PointsInTimeSeries: """ Apply a basic arithmetic operation to this series and another operand. @@ -495,10 +454,15 @@ def _apply_basic_op(self, operation, other: float | SampleSeries | PointsInTimeS """ if isinstance(other, (SampleSeries, PointsInTimeSeries)): s0, s1 = self.synchronized(other) - return PointsInTimeSeries(s0.tstarts, operation(s0.values, s1.values)) - return PointsInTimeSeries(self.tstarts, operation(self.values, other)) + value_type = self._derive_child_value_type( + operation, self.value_type, other.value_type + ) + return PointsInTimeSeries(s0.tstarts, operation(s0.values, s1.values), value_type) + return PointsInTimeSeries(self.tstarts, operation(self.values, other), self.value_type) - def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTimeSeries): + def _apply_basic_rop( + self, operation, other: float | SampleSeries | PointsInTimeSeries + ) -> PointsInTimeSeries: """ Apply a basic arithmetic operation with operands reversed. @@ -516,8 +480,11 @@ def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTime """ if isinstance(other, (SampleSeries, PointsInTimeSeries)): s0, s1 = self.synchronized(other) - return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values)) - return PointsInTimeSeries(self.tstarts, operation(other, self.values)) + value_type = self._derive_child_value_type( + operation, self.value_type, other.value_type + ) + return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values), value_type) + return PointsInTimeSeries(self.tstarts, operation(other, self.values), self.value_type) @_numeric_only def __add__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: @@ -707,36 +674,20 @@ def __repr__(self) -> str: return self.__str__() @staticmethod - def empty() -> PointsInTimeSeries: - """ - Returns an empty (numeric) PointsInTimeSeries. - - Returns - ------- - PointsInTimeSeries - Empty numeric PointsInTimeSeries object. - """ - return PointsInTimeSeries([], []) - - @staticmethod - def empty_string() -> PointsInTimeSeries: + def empty(value_type: SeriesValueType = SeriesValueType.DOUBLE) -> PointsInTimeSeries: """ - Returns an empty **string-valued** PointsInTimeSeries. + Returns an empty PointsInTimeSeries of the given value type. - An empty series has no values to infer a type from, so the constructor - defaults to numeric; this factory forces the string value type. Used for - plan-time result typing of a bare string-POI selection, where the empty - series must report the string ``dtype()`` and reject numeric-only ops - (e.g. ``mean()``) before any data is read. + Parameters + ---------- + value_type : SeriesValueType, optional + The value data type of the empty series (default ``DOUBLE``). Pass the + parent series' value type so an empty result stays correctly typed for + plan-time result-type determination. Returns ------- PointsInTimeSeries - Empty string-valued PointsInTimeSeries object. - """ - # A single-element object array makes the constructor infer string, then - # slice back to empty so no value is retained. - series = PointsInTimeSeries([], []) - series._is_string = True - series.values = np.asarray([], dtype=object) - return series + Empty PointsInTimeSeries of *value_type*. + """ + return PointsInTimeSeries([], [], value_type) diff --git a/src/impulse_query_engine/model/series/sample_series.py b/src/impulse_query_engine/model/series/sample_series.py index b149ae2e..53cf4948 100644 --- a/src/impulse_query_engine/model/series/sample_series.py +++ b/src/impulse_query_engine/model/series/sample_series.py @@ -11,6 +11,7 @@ from .intervals import Intervals from .points_in_time import PointsInTime +from .value_type import SeriesValueType FloatOrNaN = float | np.float64 @@ -39,6 +40,11 @@ def __init__(self, tstarts: Sized, tends: Sized, values: Sized): self.continuous_interval_indices = self._get_continuous_interval_indices() self.requires_deserialization = True + @property + def value_type(self) -> SeriesValueType: + """A SampleSeries carries numeric values.""" + return SeriesValueType.DOUBLE + def dtype(self): """ Returns the Spark data type for SampleSeries. diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py index cad8d2fa..a63e4175 100644 --- a/tests/impulse_query_engine/integration/poi_channel_solve_test.py +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -9,8 +9,8 @@ - ``channel_id = 91`` — a **numeric** DTC-count channel (values ``1, 2, 3``) Covers: numeric POI unweighted reductions, string POI equality + op gating, the -mix-and-match case (a SAMPLE and a POI channel in one expression), the declared-vs-actual -dtype/series-type assertion, and SAMPLE backward-compatibility. +mix-and-match case (a SAMPLE and a POI channel in one expression), and SAMPLE +backward-compatibility. """ import math @@ -106,7 +106,7 @@ def test_string_poi_numeric_reduction_rejected_at_build( q = basic_narrow_db.query dtc = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.STRING) selection = getattr(dtc, reduction)().alias("bad") - with pytest.raises(TypeError, match="string-valued"): + with pytest.raises(TypeError, match="non-numeric"): q.select(selection)._determine_result_objects_dtypes() @@ -203,34 +203,6 @@ def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db assert not math.isnan(pt[1]) -class TestDeclaredVsActual: - def test_poi_channel_declared_double_on_string_channel_raises( - self, spark: SparkSession, basic_narrow_db - ): - """Declaring ``dtype=double`` on a channel whose silver dtype is ``string`` raises - at solve time — the data stays authoritative.""" - solver = DefaultSolver(spark) - q = basic_narrow_db.query - # DTC is a numeric-less (string) channel; declaring double resolves rows - # whose value_double is all null → dtype mismatch raised in the solve UDF. - bad = q.poi_channel(channel_name="DTC", dtype=SeriesValueType.DOUBLE) - with pytest.raises(Exception, match="dtype mismatch"): - q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() - - def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): - """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. - - The SAMPLE channel's rows carry a real validity interval (``tend != tstart``), - which is the signal ``PointsInTimeSeries.from_silver`` validates a POI-declared - selector against. (A zero-duration ``tstart == tend`` row would be accepted.) - """ - solver = DefaultSolver(spark) - q = basic_narrow_db.query - bad = q.poi_channel(channel_name="Engine RPM") - with pytest.raises(Exception, match="series-type mismatch"): - q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() - - class TestBackwardCompat: def test_sample_channel_unaffected_by_poi(self, spark: SparkSession, basic_narrow_db): """An ordinary SAMPLE ``channel(...)`` selection is unchanged by POI support.""" diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 92db4312..e5fbed38 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -248,7 +248,7 @@ def test_string_poi_equality_evaluates_to_points_in_time(self, narrow_db): def test_string_poi_mean_rejected_at_build_time(self, narrow_db): dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") - with pytest.raises(TypeError, match="string-valued"): + with pytest.raises(TypeError, match="non-numeric"): dtc.mean().evaluation_type() def test_poi_selector_id_distinguishes_value_type(self): diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index 057dc5cd..d080758c 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -3,7 +3,6 @@ # pylint: disable=missing-function-docstring, redefined-outer-name import numpy as np import numpy.testing as nptest -import pandas as pd import pyspark.sql.types as T import pytest @@ -13,110 +12,43 @@ from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries -# --- from_silver (silver-slice factory + declared-vs-actual validation) ------------------------- +# --- constructor: value_type ------------------------------------------------------------------- -def _dbl(vals): - return pd.Series(vals, dtype="float64") - - -def _str(vals): - return pd.Series(vals, dtype="object") - - -def test_from_silver_numeric(): - pts = PointsInTimeSeries.from_silver( - _dbl([10, 20, 30]), - _dbl([1, 2, 3]), - _str([None, None, None]), - SeriesValueType.DOUBLE, - tend=_dbl([None, None, None]), - ) +def test_default_value_type_is_numeric(): + pts = PointsInTimeSeries([0, 1], [10, 20]) + assert pts.is_string is False assert pts.dtype() == T.ArrayType(T.ArrayType(T.DoubleType())) - nptest.assert_array_equal(pts.values, [1.0, 2.0, 3.0]) + nptest.assert_array_equal(pts.values, [10.0, 20.0]) -def test_from_silver_string(): - pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([None, None]), - _str(["P0301", "P0420"]), - SeriesValueType.STRING, - tend=_dbl([None, None]), - ) +def test_explicit_string_value_type(): + pts = PointsInTimeSeries([0, 1], ["P0301", "P0420"], SeriesValueType.STRING) + assert pts.is_string is True assert list(pts.values) == ["P0301", "P0420"] -def test_from_silver_declared_string_on_numeric_channel_raises(): - # value_string all-null => the channel is really numeric. - with pytest.raises(ValueError, match="dtype mismatch"): - PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([1, 2]), - _str([None, None]), - SeriesValueType.STRING, - tend=_dbl([None, None]), - ) - - -def test_from_silver_declared_double_on_string_channel_raises(): - # value_double all-null while value_string is populated => really a string channel. - with pytest.raises(ValueError, match="dtype mismatch"): - PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([None, None]), - _str(["P0301", "P0420"]), - SeriesValueType.DOUBLE, - tend=_dbl([None, None]), - ) - - -def test_from_silver_sample_channel_raises_series_type_mismatch(): - # A real validity interval (tend != tstart) => selector resolved to a SAMPLE channel. - with pytest.raises(ValueError, match="series-type mismatch"): - PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([1, 2]), - None, - SeriesValueType.DOUBLE, - tend=_dbl([15, 25]), - ) +def test_string_value_type_with_numeric_values_raises(): + with pytest.raises(AssertionError): + PointsInTimeSeries([0, 1], [10, 20], SeriesValueType.STRING) -def test_from_silver_zero_duration_interval_is_allowed(): - # tstart == tend is a zero-duration point (POI stored in the SAMPLE table); - # the relaxed check must NOT treat it as a series-type mismatch. - pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([1, 2]), - None, - SeriesValueType.DOUBLE, - tend=_dbl([10, 20]), - ) - nptest.assert_array_equal(pts.values, [1.0, 2.0]) +def test_double_value_type_with_string_values_raises(): + with pytest.raises(AssertionError): + PointsInTimeSeries([0, 1], ["P0301", "P0420"], SeriesValueType.DOUBLE) -def test_from_silver_null_tend_is_allowed(): - pts = PointsInTimeSeries.from_silver( - _dbl([10, 20]), - _dbl([1, 2]), - None, - SeriesValueType.DOUBLE, - tend=_dbl([None, None]), - ) - assert len(pts) == 2 +def test_empty_defaults_numeric(): + assert PointsInTimeSeries.empty().is_string is False -def test_from_silver_empty_skips_validation(): - # Nothing resolved => no mismatch can be asserted; builds an empty series. - pts = PointsInTimeSeries.from_silver( - _dbl([]), - _dbl([]), - None, - SeriesValueType.DOUBLE, - tend=_dbl([]), +def test_empty_string_typed_even_though_empty(): + empty = PointsInTimeSeries.empty(SeriesValueType.STRING) + assert empty.is_string is True + assert len(empty) == 0 + assert empty.dtype().elementType == T.StructType( + [T.StructField("tstart", T.DoubleType()), T.StructField("value", T.StringType())] ) - assert len(pts) == 0 # --- core --------------------------------------------------------------------------------------- @@ -266,43 +198,43 @@ def test_aggregations_empty(): def test_string_values_stored_as_object_with_numeric_timestamps(): - pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) - assert pts._is_string is True + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"], SeriesValueType.STRING) + assert pts.is_string is True assert pts.values.dtype == object assert pts.tstarts.dtype == np.float64 nptest.assert_array_equal(pts.values, ["P108B", "U0046", "P108B"]) def test_empty_series_defaults_to_numeric(): - # No observed value type -> numeric (backward-compatible default). - assert PointsInTimeSeries.empty()._is_string is False + # No declared value type -> numeric (backward-compatible default). + assert PointsInTimeSeries.empty().is_string is False def test_numeric_series_is_not_string(): - assert PointsInTimeSeries([0, 1], [10, 20])._is_string is False + assert PointsInTimeSeries([0, 1], [10, 20]).is_string is False def test_string_eq_scalar_returns_points_in_time(): - pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"], SeriesValueType.STRING) result = pts == "P108B" assert isinstance(result, PointsInTime) nptest.assert_array_equal(result.tstarts, [1, 3]) def test_string_ne_scalar_returns_points_in_time(): - pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"], SeriesValueType.STRING) nptest.assert_array_equal((pts != "P108B").tstarts, [2]) def test_string_eq_series_matches_on_value_and_timestamp(): - p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]) - p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"]) + p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"], SeriesValueType.STRING) + p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"], SeriesValueType.STRING) # Common timestamps {2,3}; values equal only at t=3 ("C" == "C"). nptest.assert_array_equal((p1 == p2).tstarts, [3]) def test_string_synchronized_with_sample_series_samples_values(): - pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"]) + pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"], SeriesValueType.STRING) s = SampleSeries([0, 10, 20], [10, 20, 30], [1, 2, 3]) a, b = pts.synchronized(s) nptest.assert_array_equal(a.tstarts, [5, 15, 25]) @@ -311,12 +243,12 @@ def test_string_synchronized_with_sample_series_samples_values(): def test_string_get_data_pairs_double_timestamp_with_string_value(): - pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"], SeriesValueType.STRING) assert pts.get_data() == [[1.0, "P108B"], [2.0, "U0046"]] def test_string_dtype_is_struct_of_double_and_string(): - pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"], SeriesValueType.STRING) assert pts.dtype() == T.ArrayType( T.StructType( [ @@ -339,8 +271,8 @@ def test_string_dtype_is_struct_of_double_and_string(): ], ) def test_string_arithmetic_raises(op): - pts = PointsInTimeSeries([1, 2], ["A", "B"]) - with pytest.raises(TypeError, match="string-valued"): + pts = PointsInTimeSeries([1, 2], ["A", "B"], SeriesValueType.STRING) + with pytest.raises(TypeError, match="non-numeric"): op(pts) @@ -354,21 +286,21 @@ def test_string_arithmetic_raises(op): ], ) def test_string_ordering_comparison_raises(op): - pts = PointsInTimeSeries([1, 2], ["A", "B"]) - with pytest.raises(TypeError, match="string-valued"): + pts = PointsInTimeSeries([1, 2], ["A", "B"], SeriesValueType.STRING) + with pytest.raises(TypeError, match="non-numeric"): op(pts) @pytest.mark.parametrize("reduction", ["sum", "mean", "min", "max"]) def test_string_reductions_raise(reduction): - pts = PointsInTimeSeries([1, 2], ["A", "B"]) - with pytest.raises(TypeError, match="string-valued"): + pts = PointsInTimeSeries([1, 2], ["A", "B"], SeriesValueType.STRING) + with pytest.raises(TypeError, match="non-numeric"): getattr(pts, reduction)() def test_string_count_is_allowed(): # count is structural (not value-dependent), so it works for strings. - assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]).count() == 3 + assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"], SeriesValueType.STRING).count() == 3 # --- plane_sweep -------------------------------------------------------------------------------- From 17be0884e83a39398a81eab28b0e72778a3ce9c3 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 19 Aug 2026 16:12:08 +0200 Subject: [PATCH 18/21] extended docs updated api docs --- .../metadata/time_series_expression.md | 38 +++----- .../model/series/points_in_time_series.md | 88 ++++++------------- .../model/series/sample_series.md | 9 ++ .../query_engine/tsal/core_data_model.md | 17 ++-- .../query_engine/tsal/defining_expressions.md | 31 +++++-- skills/impulse-analyze/SKILL.md | 7 ++ .../analyze/query/solvers/default_solver.py | 18 ++-- 7 files changed, 101 insertions(+), 107 deletions(-) diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index dcb00893..9f1cc130 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -11,29 +11,13 @@ class SeriesType(StrEnum) How a channel's samples are interpreted (mirrors :class:`RawEncoder`). -``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is -*valid* (reconstructed by an interpolation method, zero-order hold today), -backed by :class:`SampleSeries`. +``SAMPLE`` — the default; the time series is considered *valid* within each +``[tstart_i, tend_i)`` interval: value ``v_i`` was measured at ``tstart_i`` and no +other value was measured until ``tend_i`` (reconstructed by an interpolation method, +zero-order hold today), backed by :class:`SampleSeries`. -``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no -between-point validity, backed by :class:`PointsInTimeSeries`. - - -## SeriesValueType - -```python -class SeriesValueType(StrEnum) -``` - -The value data type of a POI channel — selects its ``poi_channels`` value - -column and which in-memory :class:`PointsInTimeSeries` variant is built. - -``DOUBLE`` — numeric points (``poi_channels.value_double``); the full -arithmetic / ordering / reduction operator set applies. - -``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); -only sampling and equality apply (see :class:`PointsInTimeSeries`). +``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their +timestamps ``(tᵢ, vᵢ)`` with no between-point validity, backed by ## TimeSeriesSelector @@ -72,7 +56,7 @@ typing and string-op gating; validated against the silver #### dtype ```python -def dtype() +def dtype() -> T.DataType ``` Returns the Spark data type. @@ -107,7 +91,7 @@ as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. #### build ```python -def build(cache: SeriesCache) +def build(cache: SeriesCache) -> SampleSeries | PointsInTimeSeries ``` Instantiate the selected series from cache data. @@ -231,7 +215,7 @@ Initialize a TimeSeriesAliasSelector. #### dtype ```python -def dtype() +def dtype() -> T.DataType ``` Returns the Spark data type. @@ -243,7 +227,7 @@ Returns the Spark data type. #### build ```python -def build(cache: SeriesCache) -> SampleSeries +def build(cache: SeriesCache) -> SampleSeries | PointsInTimeSeries ``` Build the time series from cache. @@ -254,7 +238,7 @@ Build the time series from cache. **Returns**: -`SampleSeries`: Built sample series. +`SampleSeries or PointsInTimeSeries`: Built series (a ``SampleSeries`` for the SAMPLE-only aliases used today). #### get\_required\_tag\_exprs diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index df772f7d..4452972e 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -15,7 +15,9 @@ class PointsInTimeSeries() #### \_\_init\_\_ ```python -def __init__(tstarts: Sized, values: Sized) +def __init__(tstarts: Sized, + values: Sized, + value_type: SeriesValueType = SeriesValueType.DOUBLE) ``` Initialize the PointsInTimeSeries object. @@ -24,67 +26,37 @@ A PointsInTimeSeries associates a value to each timestamp. Unlike a SampleSeries a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. -The value type (numeric vs string) is inferred from *values*. An **empty** -series has no values to infer from and therefore defaults to numeric; use -:meth:`empty_string` when an explicitly string-typed empty series is needed -(e.g. plan-time result typing of a bare string-POI selection). - **Arguments**: - `tstarts` (`Sized`): Array-like of time points. - `values` (`Sized`): Array-like of values, one per time point. +- `value_type` (`SeriesValueType`): The value data type (default ``DOUBLE``). Validated against *values* for a +non-empty series; carried explicitly so an **empty** series stays typed +correctly (there are no values to infer from). ``STRING`` series support +only sampling and equality — see the ``@_numeric_only`` methods. -#### from\_silver +#### value\_type ```python -def from_silver(cls, - tstarts: pd.Series, - values_double: pd.Series, - values_string: pd.Series | None, - value_type: SeriesValueType, - tend: pd.Series | None = None) -> PointsInTimeSeries +def value_type() -> SeriesValueType ``` -Build a POI series from a resolved silver-layer slice, validating the - -declared ``value_type`` (and, when *tend* is given, the row shape) against -the data the selector actually landed on. +This series' value data type (numeric ``DOUBLE`` or ``STRING``). -The selector drives series-type dispatch, but the silver data stays -authoritative: a ``poi_channel(...)`` selector must resolve to genuine POI -rows. This reconciliation lives here — rather than in ``__init__`` — because -it needs information a constructed point series does not carry: **both** -value columns (to tell a mis-declared dtype from a legitimately empty one) -and the ``tend`` column (to detect a SAMPLE channel). ``__init__`` stays a -thin value constructor used throughout the series algebra. -**Arguments**: +#### is\_string -- `tstarts` (`pandas.Series`): POI timestamps for the resolved rows. -- `values_double` (`pandas.Series`): The numeric value column for the resolved rows. -- `values_string` (`pandas.Series or None`): The string value column, when the frame carries one; ``None`` otherwise. -- `value_type` (`SeriesValueType`): The declared value type. ``STRING`` builds from *values_string*, any -other value builds from *values_double*. -- `tend` (`pandas.Series or None`): The validity-interval end column. A genuine POI row has a null ``tend`` -or a zero-duration interval (``tstart == tend``, should POI ever live in -the SAMPLE ``channels`` table); a real interval (``tend != tstart``) -means the selector resolved to a SAMPLE channel. ``None`` skips the -series-type check. - -**Raises**: - -- `ValueError`: On a series-type mismatch (resolved to a SAMPLE channel) or a -declared-vs-actual dtype mismatch. Fails loudly rather than silently -reading the wrong column (mirrors the unit-conversion conflict check). +```python +def is_string() -> bool +``` -**Returns**: +Whether this series carries string (rather than numeric) values. -`PointsInTimeSeries`: A string-valued series when *value_type* is ``STRING``, else numeric. #### dtype ```python -def dtype() +def dtype() -> T.DataType ``` Returns the Spark data type for PointsInTimeSeries. @@ -469,30 +441,20 @@ Returns a string representation for debugging. #### empty ```python -def empty() -> PointsInTimeSeries +def empty( + value_type: SeriesValueType = SeriesValueType.DOUBLE +) -> PointsInTimeSeries ``` -Returns an empty (numeric) PointsInTimeSeries. - -**Returns**: - -`PointsInTimeSeries`: Empty numeric PointsInTimeSeries object. - -#### empty\_string +Returns an empty PointsInTimeSeries of the given value type. -```python -def empty_string() -> PointsInTimeSeries -``` - -Returns an empty **string-valued** PointsInTimeSeries. +**Arguments**: -An empty series has no values to infer a type from, so the constructor -defaults to numeric; this factory forces the string value type. Used for -plan-time result typing of a bare string-POI selection, where the empty -series must report the string ``dtype()`` and reject numeric-only ops -(e.g. ``mean()``) before any data is read. +- `value_type` (`SeriesValueType`): The value data type of the empty series (default ``DOUBLE``). Pass the +parent series' value type so an empty result stays correctly typed for +plan-time result-type determination. **Returns**: -`PointsInTimeSeries`: Empty string-valued PointsInTimeSeries object. +`PointsInTimeSeries`: Empty PointsInTimeSeries of *value_type*. diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/sample_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/sample_series.md index 336a78ca..5043400b 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/sample_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/sample_series.md @@ -26,6 +26,15 @@ Initialize the SampleSeries object. - `tends` (`Sized`): Array-like of interval end times. - `values` (`Sized`): Array-like of sample values. +#### value\_type + +```python +def value_type() -> SeriesValueType +``` + +A SampleSeries carries numeric values. + + #### dtype ```python diff --git a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md index 95263a00..0096c3f5 100644 --- a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md +++ b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md @@ -15,7 +15,7 @@ edge detection produces `PointsInTime`, and sampling at instants produces a `Poi | `SampleSeries` | yes | yes | channel selection, arithmetic, resampling | | `Intervals` | no | yes | comparison / logical operators, edge windows | | `PointsInTime` | no | no | `rising_edges()` / `falling_edges()` | -| `PointsInTimeSeries` | yes | no | sampling a signal at instants via `where(...)` | +| `PointsInTimeSeries` | yes | no | a `poi_channel()` selection, or sampling a signal via `where(...)` | :::note Not the storage schema This page describes the **in-memory result classes** a query evaluates to. It is unrelated to the @@ -119,13 +119,20 @@ from `SampleSeries`: **a value pertains only *to* its own timestamp** and makes signal in between consecutive timestamps. There are no durations and no most-recent-value carried forward — each value stands alone at its instant. -The natural way to obtain one is to **sample a signal at specific instants** — e.g. read engine RPM -exactly at the moments the vehicle starts moving: +There are two ways to obtain one: **select a POI channel directly** with +[`poi_channel()`](defining_expressions.md#points-in-time-poi-channels) — when the source data *is* a +stream of discrete events (DTC codes, fault events) — or **sample a continuous signal at specific +instants**, e.g. read engine RPM exactly at the moments the vehicle starts moving: ```python -rpm_at_starts = eng_rpm.where(veh_spd.rising_edges()) # PointsInTimeSeries +dtc_count = db.query.poi_channel(channel_name='DTC_count') # straight from a POI channel +rpm_at_starts = eng_rpm.where(veh_spd.rising_edges()) # sampled from a signal ``` +Prefer a `PointsInTimeSeries` over a `SampleSeries` whenever a value belongs to a single instant and +carrying it forward until the next reading would be wrong — see the [`channel()` vs `poi_channel()` +decision guide](defining_expressions.md#points-in-time-poi-channels). + Because there is no validity between points, the operators differ from `SampleSeries`: - **Arithmetic** (`+ - * /`) with another `PointsInTimeSeries` aligns the two on **exactly matching @@ -140,7 +147,7 @@ Because there is no validity between points, the operators differ from `SampleSe `PointsInTimeSeries` onto their shared instants, returning value-carrying point series. - **String-valued POI series** — produced by `poi_channel(dtype='string')` (e.g. DTC fault codes) — support only equality (`==` / `!=`) and sampling (`synchronized` / `.where`); arithmetic, ordering, - and numeric reductions raise, since they are meaningless for strings. + and numeric reductions raise, since they are not implemented for strings. → API: [`PointsInTimeSeries`](../../api/impulse_query_engine/model/series/points_in_time_series.md) diff --git a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md index b134b919..3cc8ef74 100644 --- a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md +++ b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md @@ -33,12 +33,29 @@ table is configured — see [Query Solvers](../query_solvers.md#how-defaultsolve ### Points-in-Time (POI) channels -Some channels record values meaningful only **at an instant** — an ECU Diagnostic Trouble Code -(DTC), a discrete event code — with no validity in between. Select these with -`QueryBuilder.poi_channel()` instead of `channel()`. It resolves channels exactly like `channel()` -(same tag filters), but builds a [`PointsInTimeSeries`](core_data_model.md#pointsintimeseries) from -the [`poi_channels`](../../../data_model/silver_layer_schema.md#poi_channels-optional) table rather -than a `SampleSeries` from `channels`. +Not every channel is a continuously-valid signal. Some record values meaningful only **at an +instant** — an ECU Diagnostic Trouble Code (DTC), a discrete event code — with no validity in +between. Which selector you reach for follows the **nature of the data**, not the query you want to +write: + +:::tip `channel()` vs `poi_channel()` +- **`channel()` → [`SampleSeries`](core_data_model.md#sampleseries)** — a measured signal whose value + stays valid until the next sample (engine RPM, vehicle speed, coolant temperature). The last value + holds forward between samples, aggregations are duration-weighted, and the signal can be resampled + and interpolated. +- **`poi_channel()` → [`PointsInTimeSeries`](core_data_model.md#pointsintimeseries)** — discrete + events whose value exists *only at* its own instant and says nothing about the time in between (DTC + / fault codes, event logs, a count stamped at a moment). No carry-forward, no durations, unweighted + aggregations. + +**Litmus test:** *does the value still hold a moment later, until the next reading?* If yes, it's a +sample → `channel()`. If it is a momentary event, it's a POI channel → `poi_channel()`. +::: + +`poi_channel()` resolves channels exactly like `channel()` (same tag filters), but builds a +`PointsInTimeSeries` from the +[`poi_channels`](../../../data_model/silver_layer_schema.md#poi_channels-optional) table rather than a +`SampleSeries` from `channels`. ```python # numeric POI channel (default dtype='double') @@ -53,7 +70,7 @@ rpm_at_faults = eng_rpm.where(faults) # freeze-frame: RPM at each fault instan `dtype` accepts the `SeriesValueType.DOUBLE` / `SeriesValueType.STRING` enum or the plain string `'double'` / `'string'` (default `'double'`). A **string** POI channel supports only equality (`==` / `!=`) and sampling (`.where(...)`) — arithmetic, ordering, and numeric reductions raise, as -they are meaningless for string values. +they are not implemented for string values. ### Logical aliases via channel mapping diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 91907cad..2eb196d8 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -93,6 +93,13 @@ A POI channel carries values defined *only at* their timestamp (no interval). Se `poi_channel(...)` instead of `channel(...)` — identification (tags / `channel_metrics` columns) is identical; only the built series type differs: +**Which to use** — pick by the nature of the data, not the query. Use `channel()` (→ `SampleSeries`) +for a continuously-valid signal whose value holds until the next sample (RPM, speed, temperature: +carry-forward, duration-weighted aggregations). Use `poi_channel()` (→ `PointsInTimeSeries`) for +discrete events valid only at their instant (DTC / fault codes, event logs: no carry-forward, +unweighted aggregations). Litmus test: *does the value still hold a moment later, until the next +reading?* Yes → `channel()`; a momentary event → `poi_channel()`. + ```python # numeric POI channel (default dtype="double") dtc_count = db.query.poi_channel(channel_name="DTC_count") diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 143144c5..152a2c6a 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -111,7 +111,14 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.mdf) return self.mdf[idx] - def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): + def load_blob( + self, + mid, + cid, + uses_alias: bool = False, + series_type=None, + value_type=SeriesValueType.DOUBLE, + ): """ Load a time series blob from the DataFrame. @@ -156,13 +163,14 @@ def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_ # The selector's declared value type picks the column; the constructor # reconciles values-vs-type. A channel may carry both value columns — # we simply read the declared one. - vt = value_type if value_type is not None else SeriesValueType.DOUBLE - value_col = self._value_string_col if vt is SeriesValueType.STRING else self._val_col # todo extend so we have this in config what the col is + value_col = ( + self._value_string_col if value_type is SeriesValueType.STRING else self._val_col + ) if value_col is None or value_col not in s.columns: raise ValueError( - f"POI channel declared {vt} but its value column is not available" + f"POI channel declared {value_type} but its value column is not available" ) - return PointsInTimeSeries(s[self._ts_col], s[value_col], value_type=vt) + return PointsInTimeSeries(s[self._ts_col], s[value_col], value_type=value_type) values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: From fd0f418ed70c731ec48fa60e4721a760f4b6f985 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 19 Aug 2026 16:44:24 +0200 Subject: [PATCH 19/21] dropped unnecessary md file --- poi_series_integration.md | 896 -------------------------------------- 1 file changed, 896 deletions(-) delete mode 100644 poi_series_integration.md diff --git a/poi_series_integration.md b/poi_series_integration.md deleted file mode 100644 index 5c975be4..00000000 --- a/poi_series_integration.md +++ /dev/null @@ -1,896 +0,0 @@ ---- -sidebar_position: 1 -title: POI Series Integration ---- - -# Design: Integrating Points-in-Time (POI) Series into the Silver Layer - -**Status:** Proposed  ·  **Scope:** `impulse_query_engine` silver-layer -data model + `DefaultSolver` solve stage  ·  **Non-goal:** changing the -6-stage filter pipeline. - -## 1. Summary - -Impulse currently models every channel as a **sample series** — a sequence of -`[tstart, tend)` intervals over which the series is assumed to be **valid** (this -validity is what channel synchronization relies on). It does *not* intrinsically -assume the value is *held constant* over the interval: how a value is reconstructed -within `[tstart, tend)` is an **interpolation** choice. Today the only interpolation -used is **zero-order hold** (the value at `tstart` carries forward), but additional -interpolation methods could be added in the future without changing the underlying -validity model. We want to add a second kind of channel, a **Points-in-Time (POI) -Series**: a list of `(tᵢ, vᵢ)` pairs where each value is defined **only at its -timestamp** and **no assumption of validity (and hence no interpolation) is made -between two consecutive timestamps**. - -The backend model class already exists — -[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) -— and already implements arithmetic, comparisons, `synchronized` / `synchronized_all`, -and the reducing aggregations (`count`, `sum`, `mean`, `min`, `max`). The -integration work is therefore **not** about series math; it is about: - -1. **Where POI samples live in silver** (a new `poi_channels` table), and -2. **How the solver knows a channel is POI** — table membership (data in - `poi_channels` ⇒ POI) plus the query author's `poi_channel(...)` selector; no - explicit `series_type` column is needed (see [§3.2](#32-discriminator-table-membership-which-table-holds-the-channels-data)), and -3. **How the solve step builds a `PointsInTimeSeries` instead of a `SampleSeries`** - for those channels. - -The central design observation is that the entire metadata **filter pipeline is -already series-type-agnostic**, so POI support drops into the *solve* stage only. - -:::note Terminology - -- **Sample series** — the existing channel type; `[tstart, tend)` intervals over - which the series is *valid*, with values reconstructed by an interpolation method - (zero-order hold today). Backed by `SampleSeries`. -- **POI series** — the new channel type; `(tᵢ, vᵢ)` points valid *only at* their - timestamps, with no between-point validity or interpolation. Backed by - `PointsInTimeSeries`. - -::: - -### 1.1 Motivating example: ECU defect / error codes (DTCs) - -The canonical real-world POI series in vehicle testing is the stream of **defect -codes** (a.k.a. error codes, or **Diagnostic Trouble Codes — DTCs**) emitted by a -vehicle's Electronic Control Units (ECUs). When an ECU's diagnostic monitor detects -a fault — a misfire, a sensor reading out of range, a lost CAN message — it emits a -code at the **instant the fault is registered**. In a test fleet these are captured -off the CAN/UDS bus (e.g. via the `ReadDTCInformation` service, UDS `0x19`) and -logged with the timestamp at which the ECU reported them. - -A DTC event stream is a **textbook POI series**, and specifically a **string-valued** -one: - -- **Event-driven, not continuous.** A code exists *at* the moment the ECU raised it - and says **nothing** about the time between two codes. Interpolating "the value - between two error codes" is meaningless — which is exactly the POI validity model - (no between-point validity), and exactly what the held-over-interval `SampleSeries` - model would get *wrong*. -- **String values.** The standardized code is a short alphanumeric string in the - `P0301` form (1 letter for the system — **P**owertrain / **C**hassis / **B**ody / - **U**network — a generic/OEM digit, a subsystem family digit, and a 2-digit fault - index; e.g. `P0301` = cylinder-1 misfire). This is why POI channels need the - string `value_type` from [§3.4](#34-per-channel-value-dtype-double-vs-string): the - natural analysis is *equality* ("when did `P0301` occur?"), never arithmetic or - ordering on the code — matching the equality-only operator set we implement for - string POI series. - -This use case also motivates **mix-and-match** ([§2](#2-background-why-this-fits-so-cleanly)), -because DTCs are almost always analyzed **together with the continuous signals** -recorded in the same container: - -- **"Freeze-frame"-style analysis.** ECUs snapshot continuous PIDs (engine RPM, - vehicle speed, coolant temperature, …) at the instant a code is set. In Impulse - this is the exact shape of `PointValueAggregator` / a `PointsInTimeEvent`: sample a - `SampleSeries` channel (`Engine_RPM`) **at the timestamps of** a POI channel - (`DTC == "P0301"`). The POI channel supplies the instants; the sample channel - supplies the values valid at those instants — one query, one container, both series - types in the same pandas UDF. -- **Counting / windowing.** "How many `P0301` events occurred while - `Engine_RPM > 4000`?" combines a string-POI equality filter with an interval - derived from a sample series — again both series types in one expression. - -Sketched in the query API this design proposes ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), -the string DTC channel is selected with the dedicated `poi_channel(...)` method -and its `dtype`, and mixes freely with an ordinary `channel(...)` sample selection: - -```python -dtc = query.poi_channel(channel_name="DTC", dtype="string") # string POI series -rpm = query.channel(channel_name="Engine_RPM") # sample series - -# "freeze-frame": RPM at the instants DTC == "P0301" -rpm.where(dtc == "P0301") - -# equality is the only comparator defined on a string POI series (§3.4) -``` - -:::note Timestamp caveat (informative) - -DTCs do not universally carry an absolute wall-clock timestamp in the ECU fault -memory — the reliable instant usually comes from the logger/gateway that timestamps -the event when it reads the code (GPS/NTP-synced), and any per-DTC snapshot/extended -records are OEM-dependent. For Impulse this is an **ingestion** concern: whatever -timestamp the silver pipeline lands on `poi_channels.timestamp` is the instant the -engine treats as the point's `tᵢ`. It does not affect the data-model or solver -design below. - -::: - -## 2. Background: why this fits so cleanly - -The [`DefaultSolver` filter pipeline](../references/query_engine/query_solvers.md) -runs six stages, but only ever passes **identity + selector metadata** between -them: - -``` -filter_container_tags → filter_container_metrics → filter_channel_tags → -filter_channel_metrics → (alias resolution) → solve -``` - -Every stage up to `solve` produces at most -`(container_id, channel_id, selector_ids)` (plus optional unit columns). **None of -these stages read `tstart` / `tend` / `value` or make any interval-validity or -interpolation assumption.** The validity-and-interpolation semantics enter the -system in exactly one place: - -- `DefaultSolver.solve` reads the `channels` table, joins it to the channel-match - frame, and runs a grouped-map UDF (`_solve_udf`). -- Inside the UDF, `TimeSeriesCache.load_blob(...)` constructs a **`SampleSeries`** - from the `(ts, te, val)` columns. -- `TimeSeriesSelector.build(cache)` calls `cache.load_blob(...)` and returns that - `SampleSeries` to the expression tree. - -So a channel becomes a `SampleSeries` at `TimeSeriesCache.load_blob`, and nowhere -else. If we can make that one call return a `PointsInTimeSeries` for POI channels, -the rest of the engine — expression evaluation, events, aggregations — already -works, because `PointsInTimeSeries` and `SampleSeries` share the operator and -synchronization protocol, and `SampleSeries.where(PointsInTime)` / -`PointsInTimeSeries.plane_sweep` already bridge the two representations. - -**Mixing is the common case, and it is a per-container concern.** POI and sample -channels live in the **same containers**, and users routinely combine them in one -expression (e.g. `poi_channel - sample_channel`). Cross-type alignment -(`synchronized`) happens **inside** the per-container pandas UDF, on the in-memory -series objects. That imposes a hard requirement: **both series types for a container -must be present in the same UDF invocation.** A design that solved sample and POI -channels in two *separate* UDFs and unioned the results would break mixing — each -UDF would see only half of a container's channels and could not evaluate a -cross-type expression. The design below therefore feeds **one unified pandas frame -per container** (sample + POI channel data together) into a **single** grouped-map -UDF, and a unified cache builds the correct series type per channel. - -```mermaid -flowchart TB - subgraph pipeline["Filter pipeline (UNCHANGED — series-type-agnostic)"] - direction LR - A[container tags] --> B[container metrics] --> C[channel tags] --> D[channel metrics] - end - D -->|"(container_id, channel_id, selector_ids, series_type)"| J - - subgraph solve["solve stage (the ONLY place that touches series semantics)"] - direction TB - RS["read channels
(SAMPLE rows)"] --> J - RP["read poi_channels
(POI rows)"] --> J - J["union sample + POI sample data
keyed by (container_id, channel_id)
→ ONE frame per container"] - J --> U["grouped-map UDF, grouped by container_id
(both series types in the same pandas frame)"] - U --> C2["unified cache builds per channel:
SampleSeries (valid over interval; ZOH today)
or PointsInTimeSeries (valid only at points)"] - C2 --> EV["evaluate expression tree
cross-type ops align via synchronized"] - end - EV --> OUT([one wide row per container]) -``` - -## 3. Chosen design - -### 3.1 Storage: a separate `poi_channels` table - -POI samples are stored in a **new silver table `poi_channels`**, parallel to -`channels` but carrying a single timestamp (no derived `tend`, because a POI point -has no notion of a validity interval) and **two typed value columns** plus a -per-row `dtype` discriminator, since a POI value may be numeric **or** a string: - -| Column | Type | Nullable | Description | -|----------------|----------|----------|-------------------------------------------------------------------| -| `container_id` | `long` | No | Parent container identifier (join key). | -| `channel_id` | `int` | No | Channel identifier. | -| `timestamp` | `long` | No | Point timestamp (microseconds). | -| `value_double` | `double` | Yes | Value at this timestamp **when `dtype = double`**; else null. | -| `value_string` | `string` | Yes | Value at this timestamp **when `dtype = string`**; else null. | -| `dtype` | `string` | No | Value data type: `double` or `string`. Selects the value column. | - -For any given row exactly one of `value_double` / `value_string` is populated, -chosen by `dtype`. `dtype` is expected to be **constant per -`(container_id, channel_id)`** — a channel is either a numeric POI channel or a -string POI channel, not a mix (see [§3.4](#34-per-channel-value-dtype-double-vs-string)). - -`container_id` follows the same -[type rules as the rest of the silver layer](../data_model/silver_layer_schema.md) -— it may be `long` / `int` / `string`, but must be **consistent across all silver -tables** since the engine joins on it. This matches the CLAUDE.md invariant that -`container_id` / `channel_id` types are derived dynamically and never hardcoded. - -**Why a separate table rather than reusing `channels`:** - -- **No semantic overloading of `tend`.** The `channels` RLE format treats a - trailing zero-duration `[t, t)` row as a *closed endpoint* of a sample series (an - interval of validity that has collapsed to a single instant). Reusing that row - shape for a *whole* POI channel would require every - reader (the solve UDF, the RLE/interval encoders, `SampleSeries` construction) to - disambiguate "closed endpoint of a sample series" from "a genuine point". A - dedicated table keeps the two data shapes physically and semantically distinct. -- **Cleaner ingestion contract.** Producers write POI points as `(timestamp, value)` - with no obligation to synthesize a `tend`, which they cannot do correctly for POI - data anyway. -- **Minimal disturbance to the sample-series path.** The existing `channels` read - and RLE/interval encoding are untouched, and a `SAMPLE` channel still builds the - identical `SampleSeries`. The cache does gain a per-channel series-type dispatch - (required so sample and POI channels can be mixed in one UDF — see - [§4.3](#43-one-unified-per-container-frame-one-udf-one-dispatching-cache)), but the - sample branch's behavior is unchanged. - -The cost is a new configured table + a new read path + a branch in the solve -prelude — all localized to `DefaultSolver.solve` / `MeasurementDB` (see §4). - -### 3.2 Discriminator: a `series_type` column on `channel_metrics` - -:::note Implemented differently — see [§9](#9-aspects-which-differ-from-the-design) -The `series_type` column described below was **not** added. Table membership -(`channels` vs `poi_channels`) is the discriminator instead. See [§9](#9-aspects-which-differ-from-the-design). -::: - -A channel is marked POI by a **new `series_type` column on `channel_metrics`**: - -| Column | Type | Nullable | Description | -|---------------|----------|----------|--------------------------------------------------------------------| -| `series_type` | `string` | Yes | `SAMPLE` (default) or `POINTS_IN_TIME`. Null/absent ⇒ `SAMPLE`. | - -Design points: - -- **Backward compatible.** Existing tables without the column, or with `NULL`, - resolve to `SAMPLE`, so every current deployment behaves exactly as today. -- **Rides the pipeline as pass-through metadata.** `channel_metrics` is already - read in `filter_channel_metrics`; `series_type` is just one more column carried - on the channel-match rows through to `solve`. It participates in **no** filtering - decision. -- **Not `value_type`.** `channel_metrics.value_type` already exists but describes - the *value's data type* (`double`, `int`, …). Overloading it to also encode - *series semantics* would conflate two orthogonal concepts and is rejected. A new, - purpose-specific column keeps the discriminator explicit and self-documenting. -- **Introduce a `SeriesType` enum** (mirroring `RawEncoder`) so the string literals - live in one place and are referenced by `SolverConfig.series_type_col` / - the solve branch rather than being sprinkled as bare strings. - -`series_type` is added to `SolverConfig` as an internal column name property -(`series_type_col`, default `"series_type"`), so a physical layout that names the -column differently maps it via `channel_metrics.column_name_mapping` exactly like -every other column. - -### 3.3 Data model after the change - -```mermaid -erDiagram - container_metrics { - long container_id PK - } - channel_metrics { - long container_id FK - int channel_id FK - string series_type "SAMPLE | POINTS_IN_TIME (null ⇒ SAMPLE)" - } - channels { - long container_id FK - int channel_id FK - long tstart - long tend - double value - } - poi_channels { - long container_id FK - int channel_id FK - long timestamp - double value_double "when dtype = double" - string value_string "when dtype = string" - string dtype "double | string" - } - - container_metrics ||--o{ channel_metrics : container_id - channel_metrics ||--o{ channels : "SAMPLE channels" - channel_metrics ||--o{ poi_channels : "POINTS_IN_TIME channels" -``` - -A given `(container_id, channel_id)` has its samples in **exactly one** of -`channels` or `poi_channels`, selected by its `series_type` row in -`channel_metrics`. - -### 3.4 Per-channel value dtype: double vs string - -The `poi_channels.dtype` column determines which value column -(`value_double` / `value_string`) carries the point value. We treat `dtype` as a -**per-channel** property: all rows of a `(container_id, channel_id)` share one -`dtype`. This keeps a channel's value type stable, matches how measurement channels -behave in practice, and lets the solve step pick the value column **once** per -channel rather than per row. - -The two dtypes are **not** symmetric, because the backend model represents them -differently. `PointsInTimeSeries` **cannot represent string values today** — its -constructor hardcodes `np.array(values, dtype=np.float64)`, which would coerce -strings to `NaN`. We close this gap by extending the **single** -[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) -class to hold values of either kind, rather than adding a second class. - -**Chosen model change — dual value arrays + a `value_type` property:** - -- **Keep the existing `float64` value array** for numeric values (unchanged; today's - numeric behavior is preserved bit-for-bit). -- **Add a second value array of dtype `object`** to hold string values. - *(Implemented differently — a single value array whose type is inferred at - construction; see [§9](#9-aspects-which-differ-from-the-design).)* -- **Add a `value_type` property on the class** distinguishing a **numeric** from a - **string** POI series. This is the single source of truth for which value array is - populated and which operations are legal. (Constructors/factories set it; a - numeric series leaves the object array empty and vice-versa.) -- **Spark `dtype()` becomes `value_type`-aware:** `ArrayType(ArrayType(DoubleType))` - for numeric (unchanged), `ArrayType(ArrayType(StringType))` for string. - -**Operations on a string POI series (this iteration):** - -- **Only the equality comparator (`==`) is implemented.** It matches the numeric - behavior — synchronize on shared timestamps, compare values, return the - `PointsInTime` where values are equal — but over string values. - *(Implemented more permissively — both `==` and `!=` are supported for strings; - see [§9](#9-aspects-which-differ-from-the-design).)* -- **All other comparators (`<`, `<=`, `>`, `>=`) return a - `NotImplementedError`** for a string series, as do the numeric-only reductions and - arithmetic (`sum`, `mean`, `min`, `max`, `+`, `-`, `*`, `/`). These raise a clear, - explicit error rather than silently coercing to `NaN`. -- Value-type-independent operations remain valid regardless of `value_type`: - `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and the - timestamp side of `synchronized`. - -:::note "series type" appears on three distinct axes — keep them straight - -| Where | Values | Meaning | -|-------|--------|---------| -| table membership (`channels` / `poi_channels`) | sample vs POI | Which table holds the channel's data — *this* is the sample-vs-POI discriminator (no `series_type` column; see [§9](#9-aspects-which-differ-from-the-design)). | -| `poi_channels.dtype` (silver column) | `double` / `string` | A POI channel's value type — selects `value_double` vs `value_string`. | -| `PointsInTimeSeries.value_type` (class property) | numeric / string | Which in-memory value array is active and which operations are legal. | - -The middle and bottom rows are the same distinction on two sides of the Arrow -boundary: `poi_channels.dtype` on a channel becomes `PointsInTimeSeries.value_type` -on the object the cache builds for it. - -::: - -The **selectable operations are gated by `value_type`** so that, e.g., -`string_poi.mean()` fails up front (via `evaluation_type()` — see [§4.4](#44-result-typing)) -rather than producing `NaN`. - -:::note Scope check - -String POI support is the one part of this design that requires touching the -backend model (`PointsInTimeSeries`). Everything else — storage, discriminator, -pipeline, solve branch — is additive. If string POI is not needed in the first -iteration, the numeric (`double`) path can ship alone: the solver simply routes -only `dtype = double` channels and rejects (or ignores, per config) `string` -channels until the model work lands. - -::: - -### 3.5 Example: tag & metric entries for DTC POI channels - -Concrete rows for the [DTC example](#11-motivating-example-ecu-defect--error-codes-dtcs), -on an existing recording `container_id = 1`. Two POI channels are added on -`channel_id`s not used by any sample channel in that container: a **string** DTC-code -channel (`channel_id = 90`) and a **numeric** fault-occurrence-count channel -(`channel_id = 91`). - -#### Channel level — where POI-specific entries naturally live - -**Channel selection metadata.** In the EAV layout these are `channel_tags` rows -(`container_id, channel_id, key, value`); in the wide layout the same facts are -columns on `channel_metrics`. A DTC channel is selected by its `channel_name` and -described by ECU/bus context: - -| container_id | channel_id | key | value | -|--------------|------------|----------------|--------------| -| 1 | 90 | `channel_name` | `DTC` | -| 1 | 90 | `ecu` | `Engine_ECU` | -| 1 | 90 | `bus` | `CAN1` | -| 1 | 90 | `code_system` | `P` (powertrain) | -| 1 | 91 | `channel_name` | `DTC_count` | -| 1 | 91 | `ecu` | `Engine_ECU` | - -**Channel metrics** (`channel_metrics`). The **new `series_type`** marks the channel -as POI; the **existing `value_type`** records the value data type. Crucially, the -numeric statistic columns behave differently by value type — they are **undefined -(null) for a string POI channel**, and meaningful (computed over the point values, -**unweighted** — there are no durations) for a numeric one: - -| Column | DTC string channel (90) | DTC count numeric channel (91) | Notes | -|----------------|-------------------------|--------------------------------|-------| -| `series_type` | `POINTS_IN_TIME` | `POINTS_IN_TIME` | new discriminator (§3.2) | -| `value_type` | `STRING` | `DOUBLE` | pre-existing data-type column | -| `channel_name` | `DTC` | `DTC_count` | selection key (wide layout) | -| `sample_count` | `3` (three events) | `3` | number of points | -| `begin_s`/`end_s` | first/last event time | first/last event time | point extent, not a validity span | -| `min`/`max`/`mean`/`std` | **null** | computed over point values | undefined for strings; unweighted for numeric POI | -| `pz1`/`pz10`/`pz90`/`pz99` | **null** | optional | percentiles undefined for strings | -| `nan_ratio` | **null** | **null** | duration-weighted → N/A for POI | - -The per-row **`dtype`** (`string` / `double`) lives on `poi_channels`, not here (§3.1); -`series_type` on `channel_metrics` is what routes the channel to `poi_channels`. - -#### Container level — optional summaries for pre-filtering - -A container is a whole recording and owns **both** sample and POI channels, so -container-level tags/metrics are **not** POI-specific — the usual `vehicle_key`, -`brand`, `model`, `project` entries are unchanged. What POI *optionally* adds here is -**summary metadata that lets you pre-filter containers** without scanning -`poi_channels` (the same role the percentile columns play for sample channels): - -EAV `container_tags` (`container_id, key, value`): - -| container_id | key | value | Purpose | -|--------------|------------------|-------------|---------| -| 1 | `vehicle_key` | `Seat_Leon` | existing — unchanged | -| 1 | `has_dtc` | `true` | optional — "recordings that logged any fault" | -| 1 | `ecu_sw_version` | `4.11.2` | optional — correlate faults with firmware | - -Wide `container_metrics` can carry the analogous optional column -`num_dtc_events = 3` for the same pre-filtering purpose. - -These container-level additions are **purely optional and additive**: omit them and -POI channels still work; add them only to enable "find recordings where a `P0301` -occurred"-style container filters before the channel stage. A query like -`query.havingTag(has_dtc="true")` then narrows containers exactly as any other -container tag does — no POI-specific pipeline behavior. - - -## 4. Implementation plan - -The change is localized. Nothing in stages 1–5 of the pipeline changes. - -### 4.1 Config & schema - -1. `SolverConfig`: add `poi_channels: TableConfig`, add the `series_type_col` - property (`"series_type"`), and add a `poi_channels_uri` slot to - `MeasurementDBConfig` (+ `for_unity_catalog` / `for_debug` wiring, mirroring - `channels_uri`). `poi_channels_uri = None` means "no POI channels configured". -2. `MeasurementDB.poi_channels(spark)` reader, mirroring `channels(...)`. -3. `schema.py`: add a reference `POI_CHANNELS_SCHEMA` (`container_id`, `channel_id`, - `timestamp`, `value_double`, `value_string`, `dtype`) and add `series_type` to - `CHANNEL_METRICS`. As documented in CLAUDE.md these are **reference** schemas, - not enforced on read. -4. Add a `SeriesType` StrEnum (`SAMPLE`, `POINTS_IN_TIME`) next to `RawEncoder`, and - a `PoiValueType` StrEnum (`double`, `string`) for the per-row `dtype`. -5. Add `SolverConfig` internal-name properties for the new POI columns - (`poi_timestamp_col`, `poi_value_double_col`, `poi_value_string_col`, - `poi_dtype_col`) so physical layouts remap them via - `poi_channels.column_name_mapping` like every other table. -6. Extend `SolverConfig.col_map` (the short-key → column-name map handed to the UDF - cache, today `cid/ch/ts/te/val/conv`) with `series_type`, `value_string`, and - `dtype` keys so the unified cache (§4.3) can locate them in the pandas frame. -7. Add two optional fields to `TimeSeriesSelector` (`series_type`, `value_type`), - defaulting to `SAMPLE` / numeric so existing `channel(...)` selectors are - unchanged, and add `QueryBuilder.poi_channel(*, dtype=PoiValueType.double, - **kwargs)` (see [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). - -### 4.2 Query API and carrying the discriminators to solve - -#### `QueryBuilder.poi_channel(...)` - -POI channels are selected through a dedicated **`poi_channel(...)` factory method** -on `QueryBuilder`, parallel to the existing `channel(...)` / `channel_with_alias(...)`: - -```python -def poi_channel(self, *, dtype: PoiValueType = PoiValueType.double, **kwargs) -> TimeSeriesSelector: - # same tag/column matching as channel(...) — builds the selector expr from **kwargs - return TimeSeriesSelector(expr, series_type=POINTS_IN_TIME, value_type=dtype) -``` - -Design points: - -- **No new selector class.** `poi_channel` returns the **same `TimeSeriesSelector`** - that `channel(...)` returns; channel *identification* (tag/column match, - `get_selector_expr`, `required_tags`, `selector_id`, the direct/aliased split) is - identical for POI and sample channels, so there is nothing to override. The method - is a **factory**, not a subclass — it just stamps the selector with its - `series_type` (`POINTS_IN_TIME`) and the caller-declared value `dtype`. -- **Explicit intent at the call site.** `query.poi_channel(channel_name="DTC")` - reads as "this is an event stream, not a signal," and gives POI-only knobs - (the `dtype`) a natural home. `dtype` defaults to `double`, so the common numeric - case stays terse; a string DTC channel is `poi_channel(channel_name="DTC", dtype=string)`. -- **The selector now carries `series_type` + `value_type`.** `TimeSeriesSelector` - gains two optional fields (defaulting to `SAMPLE` / numeric so `channel(...)` is - unchanged). This makes the selector the **plan-time** source of truth for the - series type — which is what simplifies result typing (see [§4.4](#44-result-typing)): - `evaluation_type()` / `dtype()` and the string-op gating work **without** any - pre-pipeline `channel_metrics` lookup, and `string_poi.mean()` can be rejected at - **build time** before Spark is involved. - -:::caution Declared `dtype` is validated against the data, not trusted over it - -The user-declared `dtype` and the silver data are **two sources that must agree**. -The contract is **assertion, not authority**: - -> The check validates against the **data itself**, not a `channel_metrics.series_type` -> column (which was dropped — see [§9](#9-aspects-which-differ-from-the-design)): a POI -> point row carries a null `tend`, so a `poi_channel(...)` that resolves to -> interval-shaped rows is a SAMPLE channel, and an all-null value column exposes a -> declared/actual `dtype` mismatch. - -- The declared `series_type` / `dtype` drive **plan-time** typing and op-gating. -- At **solve time** the data remains authoritative: if the resolved channel's actual - shape **disagrees** with what the selector declared, the solver **raises a clear - error** (mirroring the existing unit-conversion conflict check), rather than silently - reading the wrong value column or overriding the data. - -This keeps the ergonomic win (no plan-time lookup, early validation) without letting -a wrong declaration silently mis-read a channel (e.g. a `dtype=double` hint on a -string channel yielding all-null `value_double`). - -::: - -#### Carrying the discriminators through the pipeline - -`filter_channel_metrics` already reads and column-maps `channel_metrics`. Include -`series_type` in the projected channel-match columns (defaulting null → `SAMPLE` -via `F.coalesce`). It travels alongside `selector_ids` with no effect on any -filter, exactly like the existing per-channel metadata. This solve-time -`series_type` (and, for POI, `dtype`) is what the **assertion check above** -validates the selector's declared values against. - -### 4.3 One unified per-container frame, one UDF, one dispatching cache - -Because sample and POI channels share containers and are mixed in a single -expression, they **must be solved together in one grouped-map UDF per container** -(see the requirement established in [§2](#2-background-why-this-fits-so-cleanly)). -The design keeps the existing single-UDF shape and makes the *cache* series-type -aware, rather than forking the UDF. - -**Step 1 — normalize both sample sources into one Spark frame.** In -`_prepare_channels_join`, read and column-map **both** tables and project them into -a common superset schema keyed by `(container_id, channel_id)`, carrying a -`series_type` discriminator (and, for POI, `dtype`): - -| Column | SAMPLE row source | POI row source | -|----------------|-----------------------|---------------------------------------| -| `container_id` | `channels` | `poi_channels` | -| `channel_id` | `channels` | `poi_channels` | -| `series_type` | `SAMPLE` | `POINTS_IN_TIME` | -| `tstart` | `channels.tstart` | `poi_channels.timestamp` | -| `tend` | `channels.tend` | `null` (POI has no validity interval) | -| `value` | `channels.value` | `poi_channels.value_double` | -| `value_string` | `null` | `poi_channels.value_string` | -| `dtype` | `null` (⇒ numeric) | `poi_channels.dtype` | - -`unionByName` the two projections into a single DataFrame, join it to the -channel-match frame on `(container_id, channel_id)`, then — exactly as today — -`groupBy(container_id).apply(udf)`. Only channels that survived the filter pipeline -are shipped, so the union stays small. A container's sample and POI rows now land in -the **same** pandas frame. - -**Step 2 — a unified cache that dispatches per channel.** Generalize -`TimeSeriesCache` (or add a `UnifiedSeriesCache` that subsumes it) so `load_blob` -inspects the channel slice's `series_type` and builds the right object: - -- `series_type == SAMPLE` → `SampleSeries(tstart, tend, value)` (today's behavior, - unchanged). -- `series_type == POINTS_IN_TIME` and `dtype == double` → numeric - `PointsInTimeSeries(tstart, value)` (the POI timestamp lives in the `tstart` - column of the unified frame). -- `series_type == POINTS_IN_TIME` and `dtype == string` → the string point series - from [§3.4](#34-per-channel-value-dtype-double-vs-string), built from - `(tstart, value_string)`. - -The cache keeps the same `(cid, ch) → (start, stop)` range-index over the sorted -frame; the only change is which columns each slice reads and which class it -instantiates. Because `series_type` and `dtype` are constant per channel, the -dispatch is decided **once** per `(cid, ch)` slice, not per row. - -**Step 3 — expression evaluation is unchanged.** `TimeSeriesSelector.build(cache)` -still just calls `cache.load_blob(...)`; it now transparently gets a `SampleSeries` -or a point series. A mixed expression such as `poi_channel - sample_channel` is -evaluated on the two in-memory objects, and `PointsInTimeSeries._apply_basic_op` -already handles the cross-type case by aligning against the `SampleSeries` at the -POI timestamps via `synchronized`. **No new math and no second UDF.** - -The `series_type` / `dtype` discriminators are carried the same pass-through way as -the existing per-channel metadata (they originate on `channel_metrics` / -`poi_channels`; see [§8](#8-open-questions)), so both the cache and the result-typing -step (§4.4) know each channel's kind without scanning its data. - -:::note Why not two UDFs? - -Splitting SAMPLE and POI into two grouped-map UDFs and unioning their **outputs** -would be simpler to write but is **incorrect** for the common mix-and-match case: -each UDF would receive only a subset of a container's channels, so an expression -referencing one channel of each type could not be evaluated — one operand would -always be missing from that UDF's frame. Unifying the **input** frame and keeping a -single UDF is what makes cross-type expressions work. - -::: - -### 4.4 Result typing - -`QueryBuilder._determine_result_objects_dtypes` builds each selection against an -`EmptyTimeSeriesCache` to learn its result `dtype`. Today `EmptyTimeSeriesCache.load_blob` -always returns an empty `SampleSeries`, so a bare POI selection would be mistyped -as `BinaryType` (the `SampleSeries` serialization dtype) instead of -`PointsInTimeSeries.dtype()` (`ArrayType(ArrayType(DoubleType))`). - -**Because the selector now carries its own `series_type` / `value_type` -([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), this resolves with -no plan-time metadata lookup.** `EmptyTimeSeriesCache.load_blob` simply consults the -calling selector and returns an empty series of the matching kind: - -- a `SAMPLE` selector → empty `SampleSeries` (today's behavior); -- a numeric POI selector → empty numeric `PointsInTimeSeries`; -- a string POI selector → empty `PointsInTimeSeries` with `value_type = string`. - -`evaluation_type()` / `dtype()` are then correct for bare POI selections and for -expressions whose output type depends on the input type — and the string-op gating -fires **at build time**: `string_poi.mean()` builds an empty string point series -whose `mean()` raises `NotImplementedError`, so the selection is rejected up front -rather than producing a silent `NaN`, before Spark is involved. - -This removes the earlier need to pre-resolve each selector's type from -`channel_metrics` and inject it into the empty cache: the declared type on the -selector *is* the plan-time source. (The silver metadata still has the final say at -solve time via the [§4.2 assertion check](#42-query-api-and-carrying-the-discriminators-to-solve).) -It also mirrors how `PointsInTimeEvent` and `PointValueAggregator` already validate -`evaluation_type()` up front, so the mechanism is consistent with existing code. - -### 4.5 `PointsInTimeSeries` model change - -The one backend-model change (see [§3.4](#34-per-channel-value-dtype-double-vs-string)): - -- Add a second value array (dtype `object`) alongside the existing `float64` array, - and a `value_type` property (numeric / string) selecting which is active. -- Constructors/factories set `value_type`: the numeric path keeps today's - `np.array(values, dtype=np.float64)`; the string path stores values as an `object` - array and leaves the numeric array empty. -- Make `dtype()` return `ArrayType(ArrayType(StringType))` when `value_type` is - string (numeric unchanged). -- Implement **`__eq__` for string series** (synchronize on timestamps → compare - string values → `PointsInTime`). Have `__ne__`, `__lt__`, `__le__`, `__gt__`, - `__ge__`, the arithmetic operators, and the numeric reductions (`sum`, `mean`, - `min`, `max`) **raise `NotImplementedError`** when `value_type` is string. -- Leave `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and - the timestamp handling in `synchronized` value-type-independent (they already are). - -### 4.6 Extend the existing test dataset with DTC POI channels - -Rather than build a bespoke POI fixture, **extend the existing session-scoped silver -dataset** so POI channels live alongside the current sample channels in the **same -containers** — this is what exercises the mix-and-match path (§4.3) end to end and -mirrors the [DTC motivating example](#11-motivating-example-ecu-defect--error-codes-dtcs). -The guiding constraint is **additive, non-destructive**: every existing test must -keep passing untouched. - -The `setup_basic_db` fixture (autouse, session-scoped) loads -`tests/unit/data/basic_narrow_csv/` into `spark_catalog.silver.*`. Use the concrete -rows from [§3.5](#35-example-tag--metric-entries-for-dtc-poi-channels) (DTC string -channel `channel_id = 90`, numeric count channel `channel_id = 91` on -`container_id = 1`) as the fixture data. The plan: - -1. **New `poi_channels` data file.** Add - `basic_narrow_csv/poi_channels.csv` with - `container_id, channel_id, timestamp, value_double, value_string, dtype` and a - couple of **DTC channels** on **existing** `container_id`s (e.g. a `DTC` string - channel with points like `(t₁, "P0301")`, `(t₂, "P0420")`, and a numeric POI - channel such as a fault-occurrence counter). Choose `channel_id`s **not already - used** by that container in `channel_data.csv` so the two sample sources stay - disjoint per the design invariant (a channel lives in exactly one of - `channels` / `poi_channels`). -2. **Append POI rows to `channel_metrics.csv`.** Add one row per new POI channel - carrying the new `series_type = POINTS_IN_TIME` column. **Backfill existing rows - with `series_type = SAMPLE`** (or leave blank and rely on the null ⇒ `SAMPLE` - default — pick one and be consistent). Existing sample channels are unaffected. -3. **Load `poi_channels` in the fixture.** Extend `setup_basic_db` to read the new - CSV and write `spark_catalog.silver.poi_channels`, and add its slot to the - `MeasurementDBConfig` used by the basic-db fixtures (`poi_channels_uri`). Because - `poi_channels_uri` defaults to `None`, **any db config that does not opt in is - unchanged**, so unrelated fixtures/tests see no difference. -4. **EAV + wide tag/metric parity.** So POI channels are *selectable* the same way - in both channel-selection modes: - - **EAV fixtures** (`setup_narrow_db`, `unit_test_csv/`): append POI rows to - `1_channel_tags.csv` (e.g. `channel_name = "DTC"`) and `1_channel_metrics.csv`, - plus any container-level tags/metrics needed, so a - `query.poi_channel(channel_name="DTC", dtype="string")` resolves the POI channel - through the pivot path (identification is identical to `channel(...)`; only the - selector's declared `series_type` / `value_type` differ — [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). - - **Wide fixtures** (`basic_narrow_csv`): the `channel_name` column already on - `channel_metrics` covers direct selection; just ensure the appended POI rows - carry a distinct `channel_name` (e.g. `"DTC"`). - -:::caution Two different columns both once called "value type" - -`basic_narrow_csv/channel_metrics.csv` **already** has a `value_type` column holding -values like `DOUBLE` (and the EAV `1_channel_metrics.csv` has `numerical`). That is -the **pre-existing** per-channel data-type column and is **not** the discriminator -this design adds. Keep them separate: - -- **existing `channel_metrics.value_type`** — untouched; describes the value data - type and is not read by the solver for routing. -- **new `channel_metrics.series_type`** — `SAMPLE` / `POINTS_IN_TIME`; routes to - `channels` vs `poi_channels` (§3.2). -- **new `poi_channels.dtype`** — `double` / `string`; selects `value_double` / - `value_string` (§3.4). - -Do **not** overload the existing `value_type` column for either new purpose — the -column names in the fixtures must stay distinct, and existing tests that read -`value_type` must be left as-is. - -::: - -**Regression guard.** Run the full existing suite after extending the fixtures and -confirm it is green *before* adding POI-specific tests (§7). Because the changes are -purely additive — new file, appended rows with a defaulting column, an opt-in table -slot — no existing assertion (row counts, computed means, dimension contents) should -move. If any does, the extension was not additive and must be corrected. - -## 5. What explicitly does **not** change - -- **The 6-stage filter pipeline.** `filter_container_tags` → - `filter_container_metrics` → `filter_channel_tags` → `filter_channel_metrics` → - alias resolution are untouched. POI channels are identified by the *same* - `TimeSeriesSelector` class, tag/column matching, and tag/metric filters as sample - channels — `poi_channel(...)` is a factory over the same selector, not a new - selection path ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). -- **`channels` table and RLE/interval encoders.** The sample-series read and - raw→interval encoding are untouched. -- **The sample-series *behavior* in the cache.** `TimeSeriesCache` gains a - per-channel dispatch (§4.3), but for a `SAMPLE` channel it builds the exact same - `SampleSeries(tstart, tend, value)` as today — the sample path's semantics and - output are unchanged. (This is a behavior guarantee, not a "no code changed" - claim: the cache does gain POI-aware branching.) -- **The single grouped-map UDF per container.** The solve stage still groups by - `container_id` and applies one UDF; POI does **not** add a second UDF or a - post-hoc union of two result sets. The input frame is widened to carry both series - types, not the execution model. -- **Persistence / gold layer.** Aggregations over POI series already reduce to - scalars (`mean`, `sum`, `count`, …) or `PointsInTime` events, which the existing - fact/dimension tables already accept (`PointsInTimeEvent`, `PointValueAggregator`). -- **`PointsInTimeSeries` for numeric (`double`) channels.** No new methods needed; - it already implements the full operator/sync/aggregation protocol for `float64` - values. (String POI is the exception — it requires the model change in - [§3.4](#34-per-channel-value-dtype-double-vs-string).) -- **`SampleSeries` interpolation semantics.** This design does not change how - sample-series values are reconstructed within `[tstart, tend)`. Zero-order hold - remains the only interpolation today; adding further interpolation methods later - is an **orthogonal** effort. The distinction that matters for POI is *validity* - (does a value exist between two timestamps at all?), not *which* interpolation is - applied where validity holds. - -## 6. Alternatives considered - -| Alternative | Why not chosen | -|-------------|----------------| -| **Store POI in `channels` with `tend == tstart`** | Overloads the "closed endpoint" meaning of zero-duration rows; forces every reader/encoder to disambiguate a whole POI channel from a sample-series endpoint. | -| **Store POI in the RAW `channels` (timestamp, value) format + a skip-encoding flag** | Couples POI to RAW mode and to the raw→interval encoder; a channel's storage shape would depend on an unrelated `data_type` setting. | -| **Overload the existing `value_type` column as the discriminator** | Conflates value *data type* with *series semantics*; two orthogonal concerns in one column, harder to reason about and to validate. | -| **A new dedicated POI solver class** | Unnecessary — the filter pipeline is shared and identical; only `load_blob` differs. A per-channel branch inside `DefaultSolver.solve` is far less code than a parallel solver. | -| **Two grouped-map UDFs (one SAMPLE, one POI), union the outputs** | **Incorrect** for the common mix-and-match case: each UDF sees only a subset of a container's channels, so an expression combining a POI and a sample channel (e.g. `poi - sample`) has a missing operand. Cross-type `synchronized` must run on both in-memory series inside **one** UDF. | - -## 7. Testing strategy - -Following the repo's fixture-reuse convention (CLAUDE.md → *Testing patterns*). -The POI tests run against the **extended shared dataset from [§4.6](#46-extend-the-existing-test-dataset-with-dtc-poi-channels)** -(DTC channels added to the existing `spark_catalog.silver.*` fixtures) rather than a -throwaway db, so they cover the real read path and the mix-and-match case: - -- Assert on **real computed values**, not row counts: e.g. a numeric POI `mean()` - equals the unweighted mean of the point values (contrast with the duration-weighted - `SampleSeries.mean()`, whose weighting follows from interval validity), proving the - between-point validity is genuinely absent. -- A **string POI** test: `query.poi_channel(channel_name="DTC", dtype="string")` - builds a `PointsInTimeSeries` with `value_type = string`; the **equality comparator** - (`== "P0301"` → `PointsInTime` on matching timestamps) and value-type-independent ops - (`count`, `to_points_in_time`, point sampling) work, while every **other comparator** - (`!=`, `<`, `<=`, `>`, `>=`), the arithmetic operators, and the numeric reductions - (`mean`, `sum`, `min`, `max`) raise `NotImplementedError` — asserted both directly on - the series object and, for a reduction inside a selection, at `evaluation_type()` - **build time** (not as a silent `NaN`, and before Spark runs). -- A **mix-and-match test (the primary correctness case)**: a single container owning - both a SAMPLE channel and a numeric POI channel, selected with `query.channel(...)` - and `query.poi_channel(...)` respectively, with **one expression referencing both** - (`rpm.where(dtc == "P0301")`, and `poi - sample`). This asserts both series land in - the *same* per-container pandas frame, are built by the unified cache, and align via - `synchronized` — the behavior a two-UDF design would break. Assert the computed - values, not just that it runs. -- A **declared-vs-actual `dtype` assertion test** ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)): - `query.poi_channel(channel_name="DTC", dtype="double")` on a channel whose silver - `dtype` is `string` (or a `poi_channel` on a `SAMPLE` channel) raises a clear error - at solve time — the data stays authoritative, the wrong declaration is not silently - honored. -- A backward-compat test: a `channel_metrics` with no `series_type` column still - solves as SAMPLE, and existing `channel(...)` selections are unaffected by the new - optional selector fields. - -## 8. Open questions - -- **Should `series_type` be validated against the presence of data in the matching - table?** (e.g. a POI-marked channel with rows only in `channels`.) Proposed: - no hard validation initially; document that the marker is authoritative and the - non-matching table is not read for that channel. -- **~~Where should the POI value `dtype` be resolved for planning?~~ (Resolved.)** - The user declares `dtype` on `query.poi_channel(...)` and the selector carries it - ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), so plan-time - result typing (§4.4) needs **no** pre-pipeline `poi_channels` / `channel_metrics` - scan. The silver `poi_channels.dtype` remains authoritative at solve time and is - validated against the declared value (assertion contract). *Remaining sub-question:* - should the engine also support **inferring** `dtype` when the user omits it (rather - than defaulting to `double`) — e.g. a cheap `distinct` on `channel_metrics` — for - callers who prefer not to declare it? Proposed: keep the explicit `double` default - for now; add inference only if a concrete need appears. -- **Enforcing the constant-`dtype` invariant.** A channel is entirely numeric or - entirely string — `dtype` is constant per `(container_id, channel_id)` by - contract. This is a settled invariant, not an open question; the only decision is - whether to *defend* it. Proposed: an optional validate-and-raise (like the - unit-conversion conflict check) that flags any channel carrying more than one - distinct `dtype`, so a malformed ingest fails loudly instead of picking an - arbitrary value column. -- **String value column when scaling.** If more non-numeric dtypes appear later - (e.g. `bool`, `int`), revisit whether a typed-column-per-dtype layout still scales - or whether a single `value` string column + cast is preferable. -- **Calculated channels producing POI output.** `solve_calculated_channels` emits a - narrow `[container_id, channel_id, tstart, tend, value]` frame. Emitting a POI - *calculated* channel would need a narrow POI shape (`timestamp, value`). Deferred — - out of scope for ingesting POI *input* series. - -## 9. Aspects which differ from the design - -A few things landed differently than sections 3–4 describe. The shipped code is the -source of truth; those sections are left as the original proposal, and each spot that -changed points here. None of these change what the feature does — they mostly remove -machinery the design added that turned out to be unnecessary once the selector became -the source of truth for a channel's series type. - -### 9.1 No `series_type` column on `channel_metrics` (§3.2) - -The design added a `series_type` marker to `channel_metrics` so the solver could tell a -POI channel from a sample channel. We dropped it. A channel's data lives in exactly one -of `channels` or `poi_channels`, so **which table it comes from already tells us the -series type** — the extra column was redundant, and nothing ever read it at solve time. -Today the only way to get a `PointsInTimeSeries` is to read from `poi_channels`, so the -table membership is a complete answer. - -### 9.2 One value array, type inferred at construction (§3.4, §4.5) - -The design proposed keeping the numeric `float64` array and adding a *second* `object` -array for strings, with a `value_type` property choosing between them. In practice -`PointsInTimeSeries` keeps a **single** value array and infers whether it's string or -numeric from the values at construction time (an `_is_string` flag). It's less -bookkeeping — there's no pair of arrays to keep in sync, one always empty — and it -reads more naturally: you build the series from whatever values you have and it figures -out its own type. An explicit `empty_string()` factory covers the one case inference -can't (an empty series has nothing to infer from). - -### 9.3 String POI also supports `!=` (§3.4, §4.5) - -The design limited string POI series to equality (`==`) and had `!=` raise alongside -the ordering and arithmetic operators. We kept `!=` too. - -### 9.4 The declared-vs-actual check reads the data shape, not a marker (§4.2) - -The design validated the selector's declared `series_type` / `dtype` against the -`channel_metrics.series_type` column. With that column gone (9.1), the solve-time check -instead looks at the **data it resolved to** - -### 9.5 Series-type dispatch is driven by the selector, not a per-row column (§4.3) - -The design's solve stage stamped `series_type` (and `dtype`) onto every channel-data -row so the cache could inspect each slice. Since the selector already knows its own -type, we pass that into `load_blob` instead and drop the per-row markers from the frame -that crosses into the pandas UDF. Only `value_string` still rides along, because that's -real data a string channel needs, not a discriminator. The result is the same object -per channel with a bit less shipped across the Arrow boundary. - -### 9.6 Enum placement (§4.1) - -Minor: the design suggested putting `SeriesType` next to `RawEncoder` in -`solver_config.py`. It lives in `time_series_expression.py` instead, next to -`TimeSeriesSelector` (which carries it) and alongside the new `PoiValueType` enum. That's -where the selector-as-source-of-truth logic reads most naturally. From 767ac6fe4d57cc4d0d90a6e684468ca67955f6fb Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Thu, 20 Aug 2026 08:23:48 +0200 Subject: [PATCH 20/21] wip --- .../metadata/time_series_expression.md | 11 ++++------ .../model/series/points_in_time_series.md | 4 +--- .../query_engine/tsal/core_data_model.md | 2 +- skills/impulse-analyze/SKILL.md | 4 ++-- skills/impulse-data-model/SKILL.md | 2 +- .../metadata/time_series_expression.py | 12 ++++------- .../model/series/points_in_time_series.py | 5 +---- .../model/series/value_type.py | 20 +++++++++++++++++++ 8 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 src/impulse_query_engine/model/series/value_type.py diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 9f1cc130..325cbef5 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -9,15 +9,12 @@ title: impulse_query_engine.analyze.metadata.time_series_expression class SeriesType(StrEnum) ``` -How a channel's samples are interpreted (mirrors :class:`RawEncoder`). +Determines how a channel's values are interpreted: -``SAMPLE`` — the default; the time series is considered *valid* within each -``[tstart_i, tend_i)`` interval: value ``v_i`` was measured at ``tstart_i`` and no -other value was measured until ``tend_i`` (reconstructed by an interpolation method, -zero-order hold today), backed by :class:`SampleSeries`. +``SAMPLE`` — the default; the time series is considered *valid* within the given intervals. +Value v_i was measured at tstart_i and no other value was measured until tend_i. -``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their -timestamps ``(tᵢ, vᵢ)`` with no between-point validity, backed by +``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their timestamps. ## TimeSeriesSelector diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index 4452972e..6b497c1a 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -450,9 +450,7 @@ Returns an empty PointsInTimeSeries of the given value type. **Arguments**: -- `value_type` (`SeriesValueType`): The value data type of the empty series (default ``DOUBLE``). Pass the -parent series' value type so an empty result stays correctly typed for -plan-time result-type determination. +- `value_type` (`SeriesValueType`): The value data type of the empty series (default ``DOUBLE``). **Returns**: diff --git a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md index 0096c3f5..e2973afe 100644 --- a/docs/impulse/docs/references/query_engine/tsal/core_data_model.md +++ b/docs/impulse/docs/references/query_engine/tsal/core_data_model.md @@ -147,7 +147,7 @@ Because there is no validity between points, the operators differ from `SampleSe `PointsInTimeSeries` onto their shared instants, returning value-carrying point series. - **String-valued POI series** — produced by `poi_channel(dtype='string')` (e.g. DTC fault codes) — support only equality (`==` / `!=`) and sampling (`synchronized` / `.where`); arithmetic, ordering, - and numeric reductions raise, since they are not implemented for strings. + and numeric reductions raise a TypeError, since they are not implemented for strings. → API: [`PointsInTimeSeries`](../../api/impulse_query_engine/model/series/points_in_time_series.md) diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 2eb196d8..62423354 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -89,8 +89,8 @@ pdf = db.query.select(eng_rpm.mean().alias("rpm_mean")).toPandas(spark, solver=D ## Selecting Points-in-Time (POI) channels -A POI channel carries values defined *only at* their timestamp (no interval). Select one with -`poi_channel(...)` instead of `channel(...)` — identification (tags / `channel_metrics` columns) is +POI channels represent time series of discrete events and are only valid at the given timestamps. +Select one with `poi_channel(...)` instead of `channel(...)` — identification (tags / `channel_metrics` columns) is identical; only the built series type differs: **Which to use** — pick by the nature of the data, not the query. Use `channel()` (→ `SampleSeries`) diff --git a/skills/impulse-data-model/SKILL.md b/skills/impulse-data-model/SKILL.md index efc29b9a..248cf2cc 100644 --- a/skills/impulse-data-model/SKILL.md +++ b/skills/impulse-data-model/SKILL.md @@ -25,7 +25,7 @@ in `source` (see `impulse-config`). | `container_metrics` | **Yes** | One row per recording — timestamps, duration, channel count, and any container-level columns. | | `channel_metrics` | **Yes** | One row per `(container_id, channel_id)` — per-channel statistics; also holds channel-selection columns (e.g. `channel_name`) in the wide model. | | `channels` | **Yes** | The time-series sample data (RLE or RAW — see below). | -| `poi_channels` | Optional | Points-in-Time (POI) channel data — a value defined *only at* its timestamp (no interval). Add to select POI channels via `poi_channel()`. | +| `poi_channels` | Optional | Points-in-Time (POI) channel data — a time series only defined at the given timestamps (discrete events). Add to select POI channels via `poi_channel()`. | | `container_tags` | Optional | EAV `(container_id, key, value)`. Add for tag-based container filtering. | | `channel_tags` | Optional | EAV `(container_id, channel_id, key, value)`. Add for EAV channel selection. | | `channel_mapping` | Optional | Logical→physical channel alias table (enables `channel_with_alias()`). | diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 223cc4db..e051020d 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -20,16 +20,12 @@ class SeriesType(StrEnum): - """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + """Determines how a channel's values are interpreted: - ``SAMPLE`` — the default; the time series is considered *valid* within each - ``[tstart_i, tend_i)`` interval: value ``v_i`` was measured at ``tstart_i`` and no - other value was measured until ``tend_i`` (reconstructed by an interpolation method, - zero-order hold today), backed by :class:`SampleSeries`. + ``SAMPLE`` — the default; the time series is considered *valid* within the given intervals. + Value v_i was measured at tstart_i and no other value was measured until tend_i. - ``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their - timestamps ``(tᵢ, vᵢ)`` with no between-point validity, backed by - :class:`PointsInTimeSeries`. + ``POINTS_IN_TIME`` — a time series of discrete events, valid *only at* their timestamps. """ SAMPLE = "SAMPLE" diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index fe4fd2aa..3ffcd1f1 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -681,10 +681,7 @@ def empty(value_type: SeriesValueType = SeriesValueType.DOUBLE) -> PointsInTimeS Parameters ---------- value_type : SeriesValueType, optional - The value data type of the empty series (default ``DOUBLE``). Pass the - parent series' value type so an empty result stays correctly typed for - plan-time result-type determination. - + The value data type of the empty series (default ``DOUBLE``). Returns ------- PointsInTimeSeries diff --git a/src/impulse_query_engine/model/series/value_type.py b/src/impulse_query_engine/model/series/value_type.py new file mode 100644 index 00000000..9f23048f --- /dev/null +++ b/src/impulse_query_engine/model/series/value_type.py @@ -0,0 +1,20 @@ +"""Value data type of a time series' samples.""" + +from enum import StrEnum + + +class SeriesValueType(StrEnum): + """The value data type of a channel's samples. + + ``DOUBLE`` — numeric values (default). ``STRING`` — string values (e.g. DTC + codes). String series support only sampling and equality (``==`` / ``!=``); + arithmetic, ordering and numeric reductions are **not implemented** for them. + """ + + DOUBLE = "double" + STRING = "string" + + @property + def is_numeric(self) -> bool: + """Whether @_numeric_only ops (arithmetic, ordering, numeric reductions) apply.""" + return self in [SeriesValueType.DOUBLE] From adc6ed1f1c071f9a95c6d2263fdffa88e628c7bb Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Thu, 20 Aug 2026 09:29:54 +0200 Subject: [PATCH 21/21] corrected comments --- .../docs/data_model/silver_layer_schema.md | 13 ++++---- .../analyze/metadata/metric_expression.md | 32 ------------------- .../analyze/query/query_builder.md | 4 +-- .../query_engine/tsal/defining_expressions.md | 16 ++++++---- skills/impulse-analyze/SKILL.md | 13 ++++---- skills/impulse-data-model/SKILL.md | 9 ++---- .../analyze/query/query_builder.py | 4 +-- 7 files changed, 26 insertions(+), 65 deletions(-) diff --git a/docs/impulse/docs/data_model/silver_layer_schema.md b/docs/impulse/docs/data_model/silver_layer_schema.md index 723e3471..d5ce7508 100644 --- a/docs/impulse/docs/data_model/silver_layer_schema.md +++ b/docs/impulse/docs/data_model/silver_layer_schema.md @@ -412,15 +412,14 @@ during raw→interval conversion (see `query_engine.raw_encoder`). ## poi_channels (optional) -**Optional** — only needed for [Points-in-Time (POI) channels](../references/query_engine/tsal/core_data_model.md#pointsintimeseries) -selected via `QueryBuilder.poi_channel()`. Omit it for sample-only data models. +**Optional** — only needed for [Points-in-Time (POI) channels](../references/query_engine/tsal/core_data_model.md#pointsintimeseries) +selected via `QueryBuilder.poi_channel()`. Omit it for sample-only data models. +Unlike `channels`, a POI channel's value is defined **only at its timestamp**. -Unlike `channels`, a POI channel's value is defined **only at its timestamp** — there is no -`[tstart, tend)` validity interval. **Table membership is the discriminator**: a -`(container_id, channel_id)` whose data lives in `poi_channels` is a POI channel; one in `channels` -is a sample channel. +**`QueryBuilder.poi_channel(...)` is the discriminator**: the user tells the engine to build a +`PointsInTimeSeries` for a specific channel by providing its name and the corresponding `dtype`. -A POI value may be numeric or a string (e.g. an ECU Diagnostic Trouble Code). The two typed value +A POI value may be numeric or a string (e.g. an ECU Diagnostic Trouble Code). The two typed value columns cover both, and **exactly one is populated per row** — chosen by the channel's declared `dtype` at query time (`poi_channel(dtype='double'|'string')`). diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/metric_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/metric_expression.md index 2ab1f91c..05746c66 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/metric_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/metric_expression.md @@ -33,22 +33,6 @@ Return a Spark SQL column expression for the selected metric. `pyspark.sql.Column`: Spark SQL column corresponding to the metric key. -#### build\_pandas - -```python -def build_pandas(df) -> pd.Series -``` - -Return a pandas Series for the selected metric from the DataFrame. - -**Arguments**: - -- `df` (`pandas.DataFrame`): DataFrame containing metric data. - -**Returns**: - -`pandas.Series`: Series corresponding to the metric key. - #### \_\_repr\_\_ ```python @@ -117,22 +101,6 @@ Build a Spark SQL expression for the metric selection. `pyspark.sql.Column`: Spark SQL column representing the metric operation. -#### build\_pandas - -```python -def build_pandas(df) -> pd.Series -``` - -Build a pandas Series for the metric operation from the given DataFrame. - -**Arguments**: - -- `df` (`pandas.DataFrame`): DataFrame containing metric data. - -**Returns**: - -`pandas.Series`: Series representing the metric operation. - #### \_\_repr\_\_ ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index c2c6c071..9649257c 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -145,9 +145,7 @@ the built object and its result dtype differ. - `dtype` (`SeriesValueType or str`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts either the enum or its string value (``"double"`` / ``"string"``). This -declared type drives plan-time result typing and string-op gating; it -is validated against the silver ``poi_channels.dtype`` at solve time -(an actual/declared mismatch raises). +declared type drives plan-time result typing and string-op gating. - `**kwargs` (`dict`): Channel tag-value pairs, matched exactly like :meth:`channel`'s. **Returns**: diff --git a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md index 3cc8ef74..92f0aa58 100644 --- a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md +++ b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md @@ -33,20 +33,22 @@ table is configured — see [Query Solvers](../query_solvers.md#how-defaultsolve ### Points-in-Time (POI) channels -Not every channel is a continuously-valid signal. Some record values meaningful only **at an +Not every channel can be interpreted as a continuous signal within its `[tstart, tend)` intervals. +Some record values meaningful only **at an instant** — an ECU Diagnostic Trouble Code (DTC), a discrete event code — with no validity in between. Which selector you reach for follows the **nature of the data**, not the query you want to write: :::tip `channel()` vs `poi_channel()` -- **`channel()` → [`SampleSeries`](core_data_model.md#sampleseries)** — a measured signal whose value - stays valid until the next sample (engine RPM, vehicle speed, coolant temperature). The last value - holds forward between samples, aggregations are duration-weighted, and the signal can be resampled - and interpolated. +- **`channel()` → [`SampleSeries`](core_data_model.md#sampleseries)** — a measured signal, **valid + within its `[tstart, tend)` intervals**: a value is measured at each `tstart` and reconstructed + between measurements by interpolation (zero-order hold today), so it has a value at *every* instant + of a valid interval. Aggregations are duration-weighted; it can be resampled. Examples: engine RPM, + vehicle speed, coolant temperature. - **`poi_channel()` → [`PointsInTimeSeries`](core_data_model.md#pointsintimeseries)** — discrete events whose value exists *only at* its own instant and says nothing about the time in between (DTC - / fault codes, event logs, a count stamped at a moment). No carry-forward, no durations, unweighted - aggregations. + / fault codes, event logs, a count stamped at a moment). No interpolation between points, no + durations, unweighted aggregations. **Litmus test:** *does the value still hold a moment later, until the next reading?* If yes, it's a sample → `channel()`. If it is a momentary event, it's a POI channel → `poi_channel()`. diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 62423354..832aae17 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -89,16 +89,17 @@ pdf = db.query.select(eng_rpm.mean().alias("rpm_mean")).toPandas(spark, solver=D ## Selecting Points-in-Time (POI) channels -POI channels represent time series of discrete events and are only valid at the given timestamps. +POI channels represent time series of discrete events and are only valid at the given timestamps. Select one with `poi_channel(...)` instead of `channel(...)` — identification (tags / `channel_metrics` columns) is identical; only the built series type differs: **Which to use** — pick by the nature of the data, not the query. Use `channel()` (→ `SampleSeries`) -for a continuously-valid signal whose value holds until the next sample (RPM, speed, temperature: -carry-forward, duration-weighted aggregations). Use `poi_channel()` (→ `PointsInTimeSeries`) for -discrete events valid only at their instant (DTC / fault codes, event logs: no carry-forward, -unweighted aggregations). Litmus test: *does the value still hold a moment later, until the next -reading?* Yes → `channel()`; a momentary event → `poi_channel()`. +for a measured signal that is valid within its `[tstart, tend)` intervals — a value measured at each +`tstart`, reconstructed between measurements by interpolation (zero-order hold today); +duration-weighted aggregations (RPM, speed, temperature). Use `poi_channel()` (→ `PointsInTimeSeries`) +for discrete events valid only at their instant, with no value in between (DTC / fault codes, event +logs; unweighted aggregations). Litmus test: *is the reading still meaningful a moment later, until +the next one?* Yes → `channel()`; a momentary event → `poi_channel()`. ```python # numeric POI channel (default dtype="double") diff --git a/skills/impulse-data-model/SKILL.md b/skills/impulse-data-model/SKILL.md index 248cf2cc..37dc46ab 100644 --- a/skills/impulse-data-model/SKILL.md +++ b/skills/impulse-data-model/SKILL.md @@ -47,13 +47,8 @@ in `source` (see `impulse-config`). `drop_implausible_data=True` (requires RAW). - Extra columns on `channels` are ignored — the engine projects down to the columns above before solving, so it is safe to keep additional bookkeeping columns on the table. -- **`poi_channels` holds Points-in-Time (POI) data** — a value defined *only at* its timestamp, with no - validity interval (`tend`), unlike sample `channels`. Schema: `(container_id long, channel_id int, - timestamp long [epoch µs], value_double double nullable, value_string string nullable)`. A channel is - a POI channel **iff** its data lives in `poi_channels` rather than `channels` — table membership *is* - the series-type discriminator; there is no `series_type`/`dtype` column. Exactly one of `value_double` - / `value_string` is populated per row (numeric vs string POI). Select with `query.poi_channel(...)` - (see `impulse-analyze`). +- **`poi_channels` holds Points-in-Time (POI) data** — a value defined *only at* its timestamp. Schema: `(container_id long, channel_id int, + timestamp long [epoch µs], value_double double nullable, value_string string nullable)`. - **Tag tables are strict EAV.** `query.channel(channel_name="Engine RPM")` looks up `channel_tags.value` where `key = 'channel_name'`. Without `channel_tags`, channel selectors match columns on `channel_metrics` instead. diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 397e1bd7..89e04a2f 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -187,9 +187,7 @@ def poi_channel( The POI channel's value data type: ``DOUBLE`` (default, numeric) or ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts either the enum or its string value (``"double"`` / ``"string"``). This - declared type drives plan-time result typing and string-op gating; it - is validated against the silver ``poi_channels.dtype`` at solve time - (an actual/declared mismatch raises). + declared type drives plan-time result typing and string-op gating. **kwargs : dict Channel tag-value pairs, matched exactly like :meth:`channel`'s.