|
31 | 31 | WithSerializationContext, |
32 | 32 | WorkflowSerializationContext, |
33 | 33 | ) |
34 | | -from temporalio.exceptions import ApplicationError, StreamError |
| 34 | +from temporalio.exceptions import ApplicationError, StreamError, StreamReplayError |
35 | 35 | from temporalio.nexus.system.stream_service import ( |
36 | 36 | Producer, |
37 | 37 | ProducerKindWorkflow, |
|
42 | 42 | from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig |
43 | 43 | from temporalio.testing import MemoryStreamStore, WorkflowEnvironment |
44 | 44 | from temporalio.worker import Replayer, Worker |
45 | | -from tests.helpers import new_worker |
| 45 | +from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker |
46 | 46 | from tests.helpers.streams import DictStreamStore |
47 | 47 | from tests.test_converter import SimpleCodec |
48 | 48 |
|
@@ -169,7 +169,7 @@ async def stream_model_output(call: ModelCall) -> None: |
169 | 169 | async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: |
170 | 170 | """Yield each chunk of the model's Stream, as the model stub of a library does.""" |
171 | 171 | stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) |
172 | | - # A read has no idle bound, so one read waits for the model until it closes the Stream. |
| 172 | + # The idle default of 1 minute is far longer than a pause of the model, so one read waits. |
173 | 173 | async for record in stream.iter(): |
174 | 174 | yield record.body |
175 | 175 |
|
@@ -1949,3 +1949,170 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli |
1949 | 1949 | _, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) |
1950 | 1950 | assert result.bodies == bodies |
1951 | 1951 | assert (result.stop, result.done) == ("Done", True) |
| 1952 | + |
| 1953 | + |
| 1954 | +# The batch window. |
| 1955 | + |
| 1956 | + |
| 1957 | +def _read_page_counts(history: WorkflowHistory) -> list[int]: |
| 1958 | + """Give the Records of each batch of the read entries of the markers, in order.""" |
| 1959 | + counts: list[int] = [] |
| 1960 | + for event in history.events: |
| 1961 | + attributes = event.marker_recorded_event_attributes |
| 1962 | + if ( |
| 1963 | + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED |
| 1964 | + or attributes.marker_name != "core_stream" |
| 1965 | + ): |
| 1966 | + continue |
| 1967 | + [payload] = attributes.details["data"].payloads |
| 1968 | + for entry in StreamMarkerData.FromString(payload.data).reads: |
| 1969 | + counts.extend(entry.page_counts) |
| 1970 | + return counts |
| 1971 | + |
| 1972 | + |
| 1973 | +@activity.defn |
| 1974 | +async def append_five_apart(stream_id: str) -> None: |
| 1975 | + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) |
| 1976 | + for index in range(5): |
| 1977 | + if index: |
| 1978 | + await asyncio.sleep(0.05) |
| 1979 | + await stream.append(f"record-{index}") |
| 1980 | + await stream.close() |
| 1981 | + |
| 1982 | + |
| 1983 | +@workflow.defn |
| 1984 | +class WindowedReadWorkflow: |
| 1985 | + @workflow.run |
| 1986 | + async def run(self, stream_id: str) -> list[str]: |
| 1987 | + # A Local Activity holds the Workflow Task open, so each batch of the read comes |
| 1988 | + # in an activation of this task and not in a later task. |
| 1989 | + writer = workflow.start_local_activity( |
| 1990 | + append_five_apart, stream_id, start_to_close_timeout=timedelta(seconds=30) |
| 1991 | + ) |
| 1992 | + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) |
| 1993 | + got = [record.body async for record in stream.iter()] |
| 1994 | + await writer |
| 1995 | + return got |
| 1996 | + |
| 1997 | + |
| 1998 | +async def _windowed_read(client: Client, window: timedelta) -> list[int]: |
| 1999 | + config = client.config() |
| 2000 | + config["stream_config"] = StreamConfig( |
| 2001 | + stores=[MemoryStreamStore()], workflow_read_batch_window=window |
| 2002 | + ) |
| 2003 | + stream_client = Client(**config) |
| 2004 | + ref = await _filled_stream(stream_client, [], close=False) |
| 2005 | + handle, got = await _run( |
| 2006 | + stream_client, WindowedReadWorkflow, ref.id, activities=[append_five_apart] |
| 2007 | + ) |
| 2008 | + assert got == [f"record-{index}" for index in range(5)] |
| 2009 | + return _read_page_counts(await handle.fetch_history()) |
| 2010 | + |
| 2011 | + |
| 2012 | +async def test_batch_window_joins_the_records_of_a_window(client: Client): |
| 2013 | + # The first batch comes immediately, and the window holds the other four for 1 s. |
| 2014 | + assert await _windowed_read(client, timedelta(seconds=1)) == [1, 4] |
| 2015 | + |
| 2016 | + |
| 2017 | +async def test_zero_batch_window_gives_more_batches(client: Client): |
| 2018 | + assert len(await _windowed_read(client, timedelta())) > 2 |
| 2019 | + |
| 2020 | + |
| 2021 | +def test_negative_batch_window_raises(): |
| 2022 | + with pytest.raises(ValueError, match="workflow_read_batch_window"): |
| 2023 | + StreamConfig( |
| 2024 | + stores=[MemoryStreamStore()], |
| 2025 | + workflow_read_batch_window=timedelta(milliseconds=-1), |
| 2026 | + ) |
| 2027 | + |
| 2028 | + |
| 2029 | +# A replay that lost the Records of a read. |
| 2030 | + |
| 2031 | + |
| 2032 | +class _ReadThenWait: |
| 2033 | + def __init__(self) -> None: |
| 2034 | + self.go = False |
| 2035 | + |
| 2036 | + async def read_then_wait(self, stream_id: str) -> list[str]: |
| 2037 | + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) |
| 2038 | + got = [record.body async for record in stream.iter(idle_timeout=timedelta())] |
| 2039 | + await workflow.wait_condition(lambda: self.go) |
| 2040 | + return got |
| 2041 | + |
| 2042 | + @workflow.signal |
| 2043 | + def proceed(self) -> None: |
| 2044 | + self.go = True |
| 2045 | + |
| 2046 | + |
| 2047 | +@workflow.defn |
| 2048 | +class ReadThenWaitWorkflow(_ReadThenWait): |
| 2049 | + @workflow.run |
| 2050 | + async def run(self, stream_id: str) -> list[str]: |
| 2051 | + return await self.read_then_wait(stream_id) |
| 2052 | + |
| 2053 | + |
| 2054 | +@workflow.defn(failure_exception_types=[StreamReplayError]) |
| 2055 | +class ReadThenWaitFailsWorkflow(_ReadThenWait): |
| 2056 | + @workflow.run |
| 2057 | + async def run(self, stream_id: str) -> list[str]: |
| 2058 | + return await self.read_then_wait(stream_id) |
| 2059 | + |
| 2060 | + |
| 2061 | +async def _replay_with_records_gone( |
| 2062 | + client: Client, workflow_type: type[Any], **worker_options: Any |
| 2063 | +) -> WorkflowHandle[Any, Any]: |
| 2064 | + """Read on one Worker, then give the run to a Worker whose Store lost the Records.""" |
| 2065 | + stream_client = _stream_client(client) |
| 2066 | + ref = await _filled_stream(stream_client, ["a", "b"]) |
| 2067 | + task_queue = f"tq-{uuid.uuid4()}" |
| 2068 | + # No cache, so the next Worker replays the History from the start. |
| 2069 | + async with new_worker( |
| 2070 | + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 |
| 2071 | + ): |
| 2072 | + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( |
| 2073 | + workflow_type.run, |
| 2074 | + ref.id, |
| 2075 | + id=f"workflow-{uuid.uuid4()}", |
| 2076 | + task_queue=task_queue, |
| 2077 | + ) |
| 2078 | + |
| 2079 | + async def read_is_in_history() -> None: |
| 2080 | + assert "core_stream" in _markers(await handle.fetch_history()) |
| 2081 | + |
| 2082 | + await assert_eventually(read_is_in_history) |
| 2083 | + # A new memory Store of the same name has none of the Records. |
| 2084 | + lost_client = _stream_client(client) |
| 2085 | + worker = new_worker( |
| 2086 | + lost_client, workflow_type, task_queue=task_queue, **worker_options |
| 2087 | + ) |
| 2088 | + async with worker: |
| 2089 | + await handle.signal("proceed") |
| 2090 | + if workflow_type is ReadThenWaitWorkflow and not worker_options: |
| 2091 | + await assert_task_fail_eventually( |
| 2092 | + handle, message_contains=f"StreamReplayError: Stream `{ref.id}`" |
| 2093 | + ) |
| 2094 | + await handle.terminate() |
| 2095 | + return handle |
| 2096 | + with pytest.raises(WorkflowFailureError) as err: |
| 2097 | + await asyncio.wait_for(handle.result(), timeout=30) |
| 2098 | + cause = err.value.cause |
| 2099 | + assert isinstance(cause, ApplicationError) |
| 2100 | + assert cause.type == "StreamReplayError" |
| 2101 | + assert f"StreamReplayError: Stream `{ref.id}`" in cause.message |
| 2102 | + return handle |
| 2103 | + |
| 2104 | + |
| 2105 | +async def test_lost_records_fail_the_task_by_default(client: Client): |
| 2106 | + await _replay_with_records_gone(client, ReadThenWaitWorkflow) |
| 2107 | + |
| 2108 | + |
| 2109 | +async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Client): |
| 2110 | + await _replay_with_records_gone( |
| 2111 | + client, |
| 2112 | + ReadThenWaitWorkflow, |
| 2113 | + workflow_failure_exception_types=[StreamReplayError], |
| 2114 | + ) |
| 2115 | + |
| 2116 | + |
| 2117 | +async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client): |
| 2118 | + await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow) |
0 commit comments