Skip to content

Commit 9ee09ba

Browse files
committed
Fix sandboxed datetime decoding with replay compatibility
1 parent 4335077 commit 9ee09ba

12 files changed

Lines changed: 777 additions & 26 deletions
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix datetime decoding failures in sandboxed workflows, including datetime values nested in collections or dataclasses and datetime-typed signals being dropped. Existing workflow histories retain their original decoding behavior during replay.

‎temporalio/_workflow_context.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
"""Private workflow compatibility flags and their runtime context."""
2+
3+
from __future__ import annotations
4+
5+
from collections.abc import Generator
6+
from contextlib import contextmanager
7+
from contextvars import ContextVar
8+
from dataclasses import dataclass
9+
from enum import IntEnum
10+
from typing import TYPE_CHECKING
11+
12+
if TYPE_CHECKING:
13+
from temporalio.worker._workflow_instance import _WorkflowInstanceImpl
14+
15+
16+
class _WorkflowLogicFlag(IntEnum):
17+
"""Flags recorded in workflow task metadata to preserve replay behavior."""
18+
19+
RAISE_ON_CANCELLING_COMPLETED_ACTIVITY = 1
20+
PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH = 2
21+
DECODE_DATETIME_PROXIES = 3
22+
23+
24+
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS: frozenset[_WorkflowLogicFlag] = frozenset(
25+
{
26+
_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH,
27+
_WorkflowLogicFlag.DECODE_DATETIME_PROXIES,
28+
}
29+
)
30+
31+
32+
@dataclass(frozen=True)
33+
class _WorkflowFlagContext:
34+
_runtime: _WorkflowInstanceImpl
35+
36+
def is_flag_enabled(self, flag: _WorkflowLogicFlag) -> bool:
37+
"""Consult live runtime state, not the state at context creation."""
38+
return self._runtime._workflow_logic_flag_enabled(flag)
39+
40+
41+
_current_workflow_flags: ContextVar[_WorkflowFlagContext | None] = ContextVar(
42+
"temporal_workflow_flags", default=None
43+
)
44+
45+
46+
@contextmanager
47+
def _workflow_flag_scope( # type:ignore[reportUnusedFunction]
48+
*,
49+
runtime: _WorkflowInstanceImpl,
50+
) -> Generator[None]:
51+
"""Expose live flag decisions without inheriting another workflow's state."""
52+
context = _WorkflowFlagContext(
53+
_runtime=runtime,
54+
)
55+
token = _current_workflow_flags.set(context)
56+
try:
57+
yield
58+
finally:
59+
_current_workflow_flags.reset(token)

‎temporalio/converter/_payload_converter.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@
4343
# Python's datetime.fromisoformat doesn't support certain formats pre-3.11
4444
from dateutil import parser # type: ignore
4545

46+
from temporalio._workflow_context import (
47+
_current_workflow_flags,
48+
_WorkflowLogicFlag,
49+
)
4650
from temporalio.converter._serialization_context import (
4751
SerializationContext,
4852
WithSerializationContext,
@@ -800,6 +804,21 @@ def from_payload(
800804
raise RuntimeError("Failed parsing") from err
801805

802806

807+
def _is_datetime_hint(hint: type) -> bool:
808+
if hint is datetime:
809+
return True
810+
context = _current_workflow_flags.get()
811+
if context is None or not context.is_flag_enabled(
812+
_WorkflowLogicFlag.DECODE_DATETIME_PROXIES
813+
):
814+
return False
815+
816+
# Importing the worker package during converter initialization creates a cycle.
817+
from temporalio.worker.workflow_sandbox._restrictions import RestrictionContext
818+
819+
return RestrictionContext.unwrap_if_proxied(hint) is datetime
820+
821+
803822
def _get_iso_datetime_parser() -> Callable[[str], datetime]:
804823
"""Isolates system version check and returns relevant datetime passer
805824
@@ -846,7 +865,7 @@ def value_to_type(
846865
# Any or primitives
847866
if hint is Any:
848867
return value
849-
elif hint is datetime:
868+
elif _is_datetime_hint(hint):
850869
if isinstance(value, str):
851870
try:
852871
return _get_iso_datetime_parser()(value)

‎temporalio/worker/_replayer.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,17 @@
2020
import temporalio.worker
2121
import temporalio.workflow
2222

23+
from .._workflow_context import (
24+
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
25+
_WorkflowLogicFlag,
26+
)
2327
from ..common import HeaderCodecBehavior
2428
from ._interceptor import Interceptor
2529
from ._worker import load_default_build_id
2630
from ._workflow import _WorkflowWorker
2731
from ._workflow_instance import (
28-
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
2932
UnsandboxedWorkflowRunner,
3033
WorkflowRunner,
31-
_WorkflowLogicFlag,
3234
)
3335
from .workflow_sandbox import SandboxedWorkflowRunner
3436

‎temporalio/worker/_worker.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
WorkerDeploymentVersion,
3131
)
3232

33+
from .._workflow_context import _WorkflowLogicFlag
3334
from ._activity import SharedStateManager, _ActivityWorker
3435
from ._interceptor import Interceptor
3536
from ._nexus import _NexusWorker
@@ -43,7 +44,6 @@
4344
PatchActivationInput,
4445
UnsandboxedWorkflowRunner,
4546
WorkflowRunner,
46-
_WorkflowLogicFlag,
4747
)
4848
from .workflow_sandbox import SandboxedWorkflowRunner
4949

‎temporalio/worker/_workflow.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@
3030
from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo
3131
from temporalio.worker.workflow_sandbox._runner import SandboxedWorkflowRunner
3232

33+
from .._workflow_context import (
34+
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
35+
_WorkflowLogicFlag,
36+
)
3337
from . import _command_aware_visitor
3438
from ._debugger import (
3539
_install_workflow_breakpoint_hook,
@@ -41,13 +45,11 @@
4145
WorkflowInterceptorClassInput,
4246
)
4347
from ._workflow_instance import (
44-
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
4548
PatchActivationInput,
4649
WorkflowInstance,
4750
WorkflowInstanceDetails,
4851
WorkflowRunner,
4952
_WorkflowExternFunctions,
50-
_WorkflowLogicFlag,
5153
)
5254

5355
logger = logging.getLogger(__name__)

‎temporalio/worker/_workflow_instance.py‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
from contextlib import contextmanager
3030
from dataclasses import dataclass, field
3131
from datetime import timedelta
32-
from enum import IntEnum
3332
from typing import (
3433
Any,
3534
Generic,
@@ -67,6 +66,11 @@
6766
)
6867
from temporalio.service import __version__
6968

69+
from .._workflow_context import (
70+
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
71+
_workflow_flag_scope,
72+
_WorkflowLogicFlag,
73+
)
7074
from ..api.failure.v1.message_pb2 import Failure
7175
from ..workflow._event_groups import (
7276
_capture_event_group_markers,
@@ -477,6 +481,12 @@ def get_thread_id(self) -> int | None:
477481

478482
def activate(
479483
self, act: temporalio.bridge.proto.workflow_activation.WorkflowActivation
484+
) -> temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion:
485+
with _workflow_flag_scope(runtime=self):
486+
return self._activate(act)
487+
488+
def _activate(
489+
self, act: temporalio.bridge.proto.workflow_activation.WorkflowActivation
480490
) -> temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion:
481491
# Reset current completion, time, and whether replaying
482492
self._current_completion = (
@@ -504,6 +514,8 @@ def activate(
504514
self._single_batch_activation = self._workflow_logic_flag_enabled(
505515
_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH
506516
)
517+
# Record this before queries can arrive, even when no input needs decoding.
518+
self._workflow_logic_flag_enabled(_WorkflowLogicFlag.DECODE_DATETIME_PROXIES)
507519
activation_err: Exception | None = None
508520
try:
509521
# Split into job sets with patches, then signals + updates, then
@@ -4251,15 +4263,3 @@ def _make_unfinished_signal_handler_message(
42514263
[{"name": name, "count": count} for name, count in names.most_common()]
42524264
)
42534265
)
4254-
4255-
4256-
class _WorkflowLogicFlag(IntEnum):
4257-
"""Flags that may be set on task/activation completion to differentiate new from old workflow behavior."""
4258-
4259-
RAISE_ON_CANCELLING_COMPLETED_ACTIVITY = 1
4260-
PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH = 2
4261-
4262-
4263-
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS: frozenset[_WorkflowLogicFlag] = frozenset(
4264-
{_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH}
4265-
)

‎tests/test_workflow_context.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
from datetime import datetime
2+
from typing import Any
3+
from unittest.mock import create_autospec
4+
5+
import pytest
6+
7+
import temporalio.converter
8+
from temporalio._workflow_context import (
9+
_current_workflow_flags,
10+
_workflow_flag_scope,
11+
_WorkflowLogicFlag,
12+
)
13+
from temporalio.worker._workflow_instance import _WorkflowInstanceImpl
14+
from temporalio.worker.workflow_sandbox._importer import Importer
15+
from temporalio.worker.workflow_sandbox._restrictions import (
16+
RestrictionContext,
17+
SandboxRestrictions,
18+
)
19+
20+
21+
class FlagRuntime:
22+
def __init__(self, enabled: bool) -> None:
23+
self.enabled = enabled
24+
self.checked: list[_WorkflowLogicFlag] = []
25+
instance = create_autospec(_WorkflowInstanceImpl, instance=True)
26+
instance._workflow_logic_flag_enabled.side_effect = (
27+
self._workflow_logic_flag_enabled
28+
)
29+
self.instance: _WorkflowInstanceImpl = instance
30+
31+
def _workflow_logic_flag_enabled(self, flag: _WorkflowLogicFlag) -> bool:
32+
self.checked.append(flag)
33+
return self.enabled
34+
35+
36+
def datetime_hint() -> type:
37+
with Importer(SandboxRestrictions.default, RestrictionContext()).applied():
38+
from datetime import datetime as sandboxed_datetime
39+
40+
return sandboxed_datetime
41+
42+
43+
def test_workflow_flag_scope_restores_context_after_exception() -> None:
44+
outer, inner = FlagRuntime(True), FlagRuntime(False)
45+
assert _current_workflow_flags.get() is None
46+
with _workflow_flag_scope(runtime=outer.instance):
47+
context = _current_workflow_flags.get()
48+
assert context is not None
49+
assert context.is_flag_enabled(_WorkflowLogicFlag.DECODE_DATETIME_PROXIES)
50+
with (
51+
pytest.raises(ValueError),
52+
_workflow_flag_scope(runtime=inner.instance),
53+
):
54+
nested = _current_workflow_flags.get()
55+
assert nested is not None
56+
assert not nested.is_flag_enabled(
57+
_WorkflowLogicFlag.DECODE_DATETIME_PROXIES
58+
)
59+
raise ValueError("test cleanup")
60+
assert _current_workflow_flags.get() is context
61+
assert _current_workflow_flags.get() is None
62+
63+
64+
def test_datetime_proxy_conversion_policy() -> None:
65+
hint = datetime_hint()
66+
assert hint is not datetime
67+
value = "2021-01-02T00:00:00+00:00"
68+
with pytest.raises(TypeError):
69+
temporalio.converter.value_to_type(hint, value)
70+
71+
disabled = FlagRuntime(False)
72+
with _workflow_flag_scope(runtime=disabled.instance):
73+
with pytest.raises(TypeError):
74+
temporalio.converter.value_to_type(hint, value)
75+
assert disabled.checked == [_WorkflowLogicFlag.DECODE_DATETIME_PROXIES]
76+
77+
enabled = FlagRuntime(True)
78+
with _workflow_flag_scope(runtime=enabled.instance):
79+
assert temporalio.converter.value_to_type(
80+
hint, value
81+
) == datetime.fromisoformat(value)
82+
assert enabled.checked == [_WorkflowLogicFlag.DECODE_DATETIME_PROXIES]
83+
84+
85+
def test_datetime_proxy_custom_converter_precedence() -> None:
86+
expected_hint = datetime_hint()
87+
88+
class CustomConverter(temporalio.converter.JSONTypeConverter):
89+
def to_typed_value(self, hint: type, value: Any) -> Any:
90+
assert hint is expected_hint
91+
return "custom"
92+
93+
runtime = FlagRuntime(True)
94+
with _workflow_flag_scope(runtime=runtime.instance):
95+
assert (
96+
temporalio.converter.value_to_type(
97+
expected_hint, "input", [CustomConverter()]
98+
)
99+
== "custom"
100+
)
101+
assert not runtime.checked

‎tests/worker/test_replayer.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010

1111
import pytest
1212

13-
import temporalio.worker._workflow_instance
1413
from temporalio import activity, workflow
14+
from temporalio._workflow_context import _WorkflowLogicFlag
1515
from temporalio.client import Client, WorkflowFailureError, WorkflowHistory
1616
from temporalio.exceptions import ApplicationError
1717
from temporalio.testing import WorkflowEnvironment
@@ -82,7 +82,6 @@ def waiting(self) -> bool:
8282
return self._waiting
8383

8484

85-
_WorkflowLogicFlag = temporalio.worker._workflow_instance._WorkflowLogicFlag
8685
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG = (
8786
_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH
8887
)

‎tests/worker/test_workflow.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@
4545
import temporalio.worker._workflow_instance
4646
import temporalio.workflow
4747
from temporalio import activity, workflow
48+
from temporalio._workflow_context import (
49+
_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS,
50+
_WorkflowLogicFlag,
51+
)
4852
from temporalio.api.common.v1 import Payload, Payloads, WorkflowExecution
4953
from temporalio.api.enums.v1 import EventType
5054
from temporalio.api.failure.v1 import Failure
@@ -1474,17 +1478,13 @@ async def run(self) -> str:
14741478
return "cancelled"
14751479

14761480

1477-
_WorkflowLogicFlag = temporalio.worker._workflow_instance._WorkflowLogicFlag
14781481
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG = (
14791482
_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH
14801483
)
14811484

14821485

14831486
def test_single_batch_workflow_activation_jobs_default_enabled() -> None:
1484-
assert (
1485-
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG
1486-
in temporalio.worker._workflow_instance._DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS
1487-
)
1487+
assert _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG in _DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS
14881488

14891489

14901490
@workflow.defn

0 commit comments

Comments
 (0)