Skip to content

Commit 88cddf9

Browse files
committed
Tested the batch window and a Workflow failure for lost Records.
A window of 1 s joins the four later Records of a read in one batch, and a zero window gives more batches. Lost Records fail the task by default, or the Workflow with the setting.
1 parent 6e32fd6 commit 88cddf9

2 files changed

Lines changed: 171 additions & 4 deletions

File tree

‎tests/contrib/redis/test_streams.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ async def run(self, stream_id: str, seen_id: str) -> list[str]:
209209
stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str)
210210
seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str)
211211
got: list[str] = []
212-
# A read has no idle bound, so one read waits across the Activity's two steps.
212+
# The idle default of 1 minute is far longer than the pause between the two steps.
213213
reading = stream.iter()
214214
async for record in reading:
215215
got.append(record.body)

‎tests/nexus/test_stream_direct.py‎

Lines changed: 170 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
WithSerializationContext,
3232
WorkflowSerializationContext,
3333
)
34-
from temporalio.exceptions import ApplicationError, StreamError
34+
from temporalio.exceptions import ApplicationError, StreamError, StreamReplayError
3535
from temporalio.nexus.system.stream_service import (
3636
Producer,
3737
ProducerKindWorkflow,
@@ -42,7 +42,7 @@
4242
from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig
4343
from temporalio.testing import MemoryStreamStore, WorkflowEnvironment
4444
from temporalio.worker import Replayer, Worker
45-
from tests.helpers import new_worker
45+
from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker
4646
from tests.helpers.streams import DictStreamStore
4747
from tests.test_converter import SimpleCodec
4848

@@ -169,7 +169,7 @@ async def stream_model_output(call: ModelCall) -> None:
169169
async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]:
170170
"""Yield each chunk of the model's Stream, as the model stub of a library does."""
171171
stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str)
172-
# A read has no idle bound, so one read waits for the model until it closes the Stream.
172+
# The idle default of 1 minute is far longer than a pause of the model, so one read waits.
173173
async for record in stream.iter():
174174
yield record.body
175175

@@ -1949,3 +1949,170 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli
19491949
_, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id))
19501950
assert result.bodies == bodies
19511951
assert (result.stop, result.done) == ("Done", True)
1952+
1953+
1954+
# The batch window.
1955+
1956+
1957+
def _read_page_counts(history: WorkflowHistory) -> list[int]:
1958+
"""Give the Records of each batch of the read entries of the markers, in order."""
1959+
counts: list[int] = []
1960+
for event in history.events:
1961+
attributes = event.marker_recorded_event_attributes
1962+
if (
1963+
event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED
1964+
or attributes.marker_name != "core_stream"
1965+
):
1966+
continue
1967+
[payload] = attributes.details["data"].payloads
1968+
for entry in StreamMarkerData.FromString(payload.data).reads:
1969+
counts.extend(entry.page_counts)
1970+
return counts
1971+
1972+
1973+
@activity.defn
1974+
async def append_five_apart(stream_id: str) -> None:
1975+
stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str)
1976+
for index in range(5):
1977+
if index:
1978+
await asyncio.sleep(0.05)
1979+
await stream.append(f"record-{index}")
1980+
await stream.close()
1981+
1982+
1983+
@workflow.defn
1984+
class WindowedReadWorkflow:
1985+
@workflow.run
1986+
async def run(self, stream_id: str) -> list[str]:
1987+
# A Local Activity holds the Workflow Task open, so each batch of the read comes
1988+
# in an activation of this task and not in a later task.
1989+
writer = workflow.start_local_activity(
1990+
append_five_apart, stream_id, start_to_close_timeout=timedelta(seconds=30)
1991+
)
1992+
stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str)
1993+
got = [record.body async for record in stream.iter()]
1994+
await writer
1995+
return got
1996+
1997+
1998+
async def _windowed_read(client: Client, window: timedelta) -> list[int]:
1999+
config = client.config()
2000+
config["stream_config"] = StreamConfig(
2001+
stores=[MemoryStreamStore()], workflow_read_batch_window=window
2002+
)
2003+
stream_client = Client(**config)
2004+
ref = await _filled_stream(stream_client, [], close=False)
2005+
handle, got = await _run(
2006+
stream_client, WindowedReadWorkflow, ref.id, activities=[append_five_apart]
2007+
)
2008+
assert got == [f"record-{index}" for index in range(5)]
2009+
return _read_page_counts(await handle.fetch_history())
2010+
2011+
2012+
async def test_batch_window_joins_the_records_of_a_window(client: Client):
2013+
# The first batch comes immediately, and the window holds the other four for 1 s.
2014+
assert await _windowed_read(client, timedelta(seconds=1)) == [1, 4]
2015+
2016+
2017+
async def test_zero_batch_window_gives_more_batches(client: Client):
2018+
assert len(await _windowed_read(client, timedelta())) > 2
2019+
2020+
2021+
def test_negative_batch_window_raises():
2022+
with pytest.raises(ValueError, match="workflow_read_batch_window"):
2023+
StreamConfig(
2024+
stores=[MemoryStreamStore()],
2025+
workflow_read_batch_window=timedelta(milliseconds=-1),
2026+
)
2027+
2028+
2029+
# A replay that lost the Records of a read.
2030+
2031+
2032+
class _ReadThenWait:
2033+
def __init__(self) -> None:
2034+
self.go = False
2035+
2036+
async def read_then_wait(self, stream_id: str) -> list[str]:
2037+
stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str)
2038+
got = [record.body async for record in stream.iter(idle_timeout=timedelta())]
2039+
await workflow.wait_condition(lambda: self.go)
2040+
return got
2041+
2042+
@workflow.signal
2043+
def proceed(self) -> None:
2044+
self.go = True
2045+
2046+
2047+
@workflow.defn
2048+
class ReadThenWaitWorkflow(_ReadThenWait):
2049+
@workflow.run
2050+
async def run(self, stream_id: str) -> list[str]:
2051+
return await self.read_then_wait(stream_id)
2052+
2053+
2054+
@workflow.defn(failure_exception_types=[StreamReplayError])
2055+
class ReadThenWaitFailsWorkflow(_ReadThenWait):
2056+
@workflow.run
2057+
async def run(self, stream_id: str) -> list[str]:
2058+
return await self.read_then_wait(stream_id)
2059+
2060+
2061+
async def _replay_with_records_gone(
2062+
client: Client, workflow_type: type[Any], **worker_options: Any
2063+
) -> WorkflowHandle[Any, Any]:
2064+
"""Read on one Worker, then give the run to a Worker whose Store lost the Records."""
2065+
stream_client = _stream_client(client)
2066+
ref = await _filled_stream(stream_client, ["a", "b"])
2067+
task_queue = f"tq-{uuid.uuid4()}"
2068+
# No cache, so the next Worker replays the History from the start.
2069+
async with new_worker(
2070+
stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0
2071+
):
2072+
handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow(
2073+
workflow_type.run,
2074+
ref.id,
2075+
id=f"workflow-{uuid.uuid4()}",
2076+
task_queue=task_queue,
2077+
)
2078+
2079+
async def read_is_in_history() -> None:
2080+
assert "core_stream" in _markers(await handle.fetch_history())
2081+
2082+
await assert_eventually(read_is_in_history)
2083+
# A new memory Store of the same name has none of the Records.
2084+
lost_client = _stream_client(client)
2085+
worker = new_worker(
2086+
lost_client, workflow_type, task_queue=task_queue, **worker_options
2087+
)
2088+
async with worker:
2089+
await handle.signal("proceed")
2090+
if workflow_type is ReadThenWaitWorkflow and not worker_options:
2091+
await assert_task_fail_eventually(
2092+
handle, message_contains=f"StreamReplayError: Stream `{ref.id}`"
2093+
)
2094+
await handle.terminate()
2095+
return handle
2096+
with pytest.raises(WorkflowFailureError) as err:
2097+
await asyncio.wait_for(handle.result(), timeout=30)
2098+
cause = err.value.cause
2099+
assert isinstance(cause, ApplicationError)
2100+
assert cause.type == "StreamReplayError"
2101+
assert f"StreamReplayError: Stream `{ref.id}`" in cause.message
2102+
return handle
2103+
2104+
2105+
async def test_lost_records_fail_the_task_by_default(client: Client):
2106+
await _replay_with_records_gone(client, ReadThenWaitWorkflow)
2107+
2108+
2109+
async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Client):
2110+
await _replay_with_records_gone(
2111+
client,
2112+
ReadThenWaitWorkflow,
2113+
workflow_failure_exception_types=[StreamReplayError],
2114+
)
2115+
2116+
2117+
async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client):
2118+
await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow)

0 commit comments

Comments
 (0)