Skip to content

Commit 84f1587

Browse files
committed
Added Workflow Streams, which write through Local Activities.
Each write has the epoch of the Workflow Task that runs it, so the Store withdraws the writes of a failed task. A close is a regular Activity, which runs only after its Workflow Task commits.
1 parent b5bb616 commit 84f1587

14 files changed

Lines changed: 3069 additions & 6 deletions

File tree

‎temporalio/common/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
from temporalio.common._streams._plugin import StreamBackend
6969
from temporalio.common._streams._reader import Subscription
7070
from temporalio.common._streams._stream_client import ClientStream, ClientTopic
71+
from temporalio.common._streams._workflow import WorkflowStream, WorkflowTopic
7172

7273
__all__ = [
7374
"ActivityIDConflictPolicy",
@@ -135,6 +136,8 @@
135136
"WorkerDeploymentVersion",
136137
"WorkflowIDConflictPolicy",
137138
"WorkflowIDReusePolicy",
139+
"WorkflowStream",
140+
"WorkflowTopic",
138141
"defn",
139142
"topic",
140143
"with_name",
@@ -1623,6 +1626,8 @@ def wait_sync(self, timeout: float | None = None) -> None:
16231626
"Subscription",
16241627
"Superseded",
16251628
"TopicDefinition",
1629+
"WorkflowStream",
1630+
"WorkflowTopic",
16261631
"defn",
16271632
"topic",
16281633
"with_name",

‎temporalio/common/_streams/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
from temporalio.common._streams._plugin import StreamBackend
3535
from temporalio.common._streams._reader import Subscription
3636
from temporalio.common._streams._stream_client import ClientStream, ClientTopic
37+
from temporalio.common._streams._workflow import WorkflowStream, WorkflowTopic
3738

3839
__all__ = [
3940
"END",
@@ -63,6 +64,8 @@
6364
"Subscription",
6465
"Superseded",
6566
"TopicDefinition",
67+
"WorkflowStream",
68+
"WorkflowTopic",
6669
"defn",
6770
"topic",
6871
"with_name",
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
"""The Stream state of one Activity attempt."""
2+
3+
from __future__ import annotations
4+
5+
import asyncio
6+
import contextvars
7+
from dataclasses import dataclass, field
8+
from typing import Any
9+
10+
import temporalio.api.stream.v1 as stream_v1
11+
import temporalio.converter
12+
from temporalio.common._streams._backend import BackendCalls
13+
from temporalio.common._streams._writer import StreamWriter
14+
15+
16+
@dataclass
17+
class ActivityStreams:
18+
"""What a Stream backend gives the Activity handles for one Activity attempt."""
19+
20+
backend: BackendCalls
21+
converter: temporalio.converter.DataConverter
22+
namespace: str
23+
producer: stream_v1.Producer
24+
loop: asyncio.AbstractEventLoop
25+
backend_name: str = ""
26+
# Whether the Worker's backend is the default, which a handle with no backend name uses.
27+
default: bool = False
28+
# The attempt's Workflow id, which a close sends so the Store can check the Stream's owner.
29+
workflow_id: str | None = None
30+
# A Local Activity's writes use these to withdraw the run's orphaned Records.
31+
previous_completed_task: stream_v1.WorkflowTaskEpoch | None = None
32+
failed_tasks: tuple[int, ...] = ()
33+
writers: dict[str, StreamWriter] = field(default_factory=dict)
34+
# The attempt's BufferedPublishers, which the end of the attempt flushes.
35+
publishers: list[Any] = field(default_factory=list)
36+
37+
def writer(self, stream_id: str) -> StreamWriter:
38+
"""The attempt's writer for a Stream, one per Stream."""
39+
writer = self.writers.get(stream_id)
40+
if writer is None:
41+
42+
async def known_id() -> str:
43+
return stream_id
44+
45+
writer = StreamWriter(
46+
writers=self.backend.writers(self.namespace, self.converter),
47+
stream_id=known_id,
48+
producer=self.producer,
49+
free=self.backend.free_writer,
50+
previous_completed_task=self.previous_completed_task,
51+
failed_tasks=self.failed_tasks,
52+
)
53+
self.writers[stream_id] = writer
54+
return writer
55+
56+
async def close(self) -> None:
57+
"""Flush the attempt's publishers, then close every writer of the attempt."""
58+
errors = []
59+
for publisher in self.publishers:
60+
try:
61+
await publisher._flush_at_attempt_end()
62+
except Exception as err:
63+
errors.append(err)
64+
for writer in self.writers.values():
65+
try:
66+
await writer.close()
67+
except Exception as err:
68+
errors.append(err)
69+
if errors:
70+
raise errors[0]
71+
72+
73+
current: contextvars.ContextVar[ActivityStreams | None] = contextvars.ContextVar(
74+
"__temporal_activity_streams", default=None
75+
)

0 commit comments

Comments
 (0)