Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/cell/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "pycrdt >=0.12.44,<0.13.0",]
dependencies = [ "txl ==0.3.3", "pycrdt >=0.14.1,<0.15.0",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
33 changes: 19 additions & 14 deletions plugins/cell/txl_cell/main.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import json
import math
from functools import partial
from importlib.metadata import entry_points
from typing import Any

from anyio import create_task_group, sleep
from anyioutils import Queue, create_task
from anyio import create_memory_object_stream, create_task_group, sleep
from fps import Module
from pycrdt import Doc, Map, MapEvent, Text
from rich.text import Text as RichText
Expand All @@ -12,6 +13,7 @@
from textual.widgets import Static

from txl.base import Cell, CellFactory, Contents, Kernel, Widgets
from txl.stapled import StapledObjectStream
from txl.text_input import TextInput

YDOCS = {ep.name: ep.load() for ep in entry_points(group="ypywidgets")}
Expand Down Expand Up @@ -75,23 +77,27 @@ def __init__(
self.update(mount=False)
self.ycell.observe_deep(self.on_change)
self.styles.height = "auto"
self.cell_change_events = Queue()
self.widget_change_events = Queue()
create_task(self.observe_cell_changes(), task_group)
create_task(self.observe_widget_changes(), task_group)
self.cell_change_events = StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
)
self.widget_change_events = StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
)
task_group.create_task(self.observe_cell_changes())
task_group.create_task(self.observe_widget_changes())

def on_click(self):
self.clicked = True

def on_change(self, events):
self.cell_change_events.put_nowait(events)
self.cell_change_events.send_nowait(events)

def on_widget_change(self, ydoc, event):
self.widget_change_events.put_nowait((ydoc, event))
self.widget_change_events.send_nowait((ydoc, event))

async def observe_widget_changes(self):
while True:
ydoc, event = await self.widget_change_events.get()
ydoc, event = await self.widget_change_events.receive()
model_name = event.delta[0]["insert"]
model = YDOCS[f"{model_name}Model"](ydoc=ydoc)
widget = YDOCS[f"txl_{model_name}"](model)
Expand All @@ -100,7 +106,7 @@ async def observe_widget_changes(self):

async def observe_cell_changes(self):
while True:
events = await self.cell_change_events.get()
events = await self.cell_change_events.receive()
for event in events:
if isinstance(event, MapEvent):
if "execution_state" in event.keys:
Expand Down Expand Up @@ -150,9 +156,8 @@ async def observe_cell_changes(self):
# this is a widget
is_widget = True
room_id = f"ywidget:{inserted.guid}"
create_task(
self.task_group.create_task(
self.contents.websocket_provider(room_id, inserted),
self.task_group,
)
inserted["_model_name"] = model_name = Text()
model_name.observe(partial(
Expand Down Expand Up @@ -211,7 +216,7 @@ def update(self, mount: bool = True):
language=language,
show_border=self.show_border,
)
create_task(self.source.start(), self.task_group)
self.task_group.create_task(self.source.start())
if mount:
self.mount(self.source)

Expand All @@ -227,7 +232,7 @@ def get_output_widget(self, output):
guid = output["guid"]
ywidget_doc = Doc()
room_id = f"ywidget:{guid}"
create_task(self.contents.websocket_provider(room_id, ywidget_doc), self.task_group)
self.task_group.create_task(self.contents.websocket_provider(room_id, ywidget_doc))
ywidget_doc["_model_name"] = model_name = Text()
model_name.observe(partial(self.on_widget_change, ywidget_doc))
return
Expand Down
2 changes: 1 addition & 1 deletion plugins/console/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "jupyter-ydoc >=3.0.2,<4.0.0",]
dependencies = [ "txl ==0.3.3", "jupyter-ydoc >=4.1.1,<5.0.0",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
2 changes: 1 addition & 1 deletion plugins/kernel/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "python-dateutil >=2.8.2", "pycrdt >=0.12.44,<0.13.0",]
dependencies = [ "txl ==0.3.3", "python-dateutil >=2.8.2", "pycrdt >=0.14.1,<0.15.0",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
48 changes: 30 additions & 18 deletions plugins/kernel/txl_kernel/driver.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import math
import time
from typing import Dict
from typing import Any, Dict

from anyio import move_on_after
from anyioutils import Event, Queue, create_task
from anyio import Event, create_memory_object_stream, move_on_after
from fps import Signal
from pycrdt import Array, Map

from txl.stapled import StapledObjectStream

from .message import create_message


Expand All @@ -31,8 +33,8 @@ def send(self, buffers):
buffers=buffers,
)
self.msg_cnt += 1
create_task(
self.send_message(msg, self.shell_channel, change_date_to_str=True), self.task_group
self.task_group.create_task(
self.send_message(msg, self.shell_channel, change_date_to_str=True)
)


Expand All @@ -41,10 +43,12 @@ def __init__(self, task_group):
self.task_group = task_group
self.busy = Signal[bool]()
self.msg_cnt = 0
self.execute_requests: Dict[str, Dict[str, Queue]] = {}
self.recv_queue = Queue()
self.execute_requests: Dict[str, Dict[str, StapledObjectStream]] = {}
self.recv_queue = StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
)
self.started = Event()
create_task(self.recv(), task_group)
task_group.create_task(self.recv())

def create_message(self, *args, **kwargs):
return create_message(*args, **kwargs)
Expand All @@ -62,26 +66,30 @@ async def wait_for_ready(self, timeout=float("inf")):
await self.send_message(msg, self.shell_channel, change_date_to_str=True)
msg_id = msg["header"]["msg_id"]
self.execute_requests[msg_id] = {
"iopub": Queue(),
"shell": Queue(),
"iopub": StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
),
"shell": StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
),
}
with move_on_after(new_timeout) as scope:
msg = await self.execute_requests[msg_id]["shell"].get()
msg = await self.execute_requests[msg_id]["shell"].receive()
if scope.cancelled_caught:
del self.execute_requests[msg_id]
error_message = f"Kernel didn't respond in {timeout} seconds"
raise RuntimeError(error_message)
if msg["header"]["msg_type"] == "kernel_info_reply":
with move_on_after(0.2) as scope:
msg = await self.execute_requests[msg_id]["iopub"].get()
msg = await self.execute_requests[msg_id]["iopub"].receive()
if not scope.cancelled_caught:
break
del self.execute_requests[msg_id]
new_timeout = deadline_to_timeout(deadline)

async def recv(self):
while True:
msg = await self.recv_queue.get()
msg = await self.recv_queue.receive()
channel = msg.pop("channel")
msg_type = msg["header"]["msg_type"]
if msg_type == "comm_open":
Expand All @@ -108,7 +116,7 @@ async def recv(self):
if msg_id in self.execute_requests:
# msg["header"] = str_to_date(msg["header"])
# msg["parent_header"] = str_to_date(msg["parent_header"])
self.execute_requests[msg_id][channel].put_nowait(msg)
self.execute_requests[msg_id][channel].send_nowait(msg)

async def execute(
self,
Expand Down Expand Up @@ -136,15 +144,19 @@ async def execute(
msg_id = msg["header"]["msg_id"]
self.msg_cnt += 1
self.execute_requests[msg_id] = {
"iopub": Queue(),
"shell": Queue(),
"iopub": StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
),
"shell": StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
),
}
await self.send_message(msg, self.shell_channel, change_date_to_str=True)
if wait_for_executed:
deadline = time.monotonic() + timeout
while True:
with move_on_after(deadline_to_timeout(deadline)) as scope:
msg = await self.execute_requests[msg_id]["iopub"].get()
msg = await self.execute_requests[msg_id]["iopub"].receive()
if scope.cancelled_caught:
del self.execute_requests[msg_id]
error_message = f"Kernel didn't respond in {timeout} seconds"
Expand All @@ -156,7 +168,7 @@ async def execute(
):
break
with move_on_after(deadline_to_timeout(deadline)) as scope:
msg = await self.execute_requests[msg_id]["shell"].get()
msg = await self.execute_requests[msg_id]["shell"].receive()
if scope.cancelled_caught:
del self.execute_requests[msg_id]
error_message = f"Kernel didn't respond in {timeout} seconds"
Expand Down
2 changes: 1 addition & 1 deletion plugins/local_contents/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "anyio >=3.7.0,<5", "jupyter-ydoc >=3.0.2,<4.0.0",]
dependencies = [ "txl ==0.3.3", "anyio >=4.14.2,<5", "jupyter-ydoc >=4.1.1,<5.0.0",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
2 changes: 1 addition & 1 deletion plugins/local_kernels/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "txl_kernel", "pyzmq >=24.0.1", "ipykernel",]
dependencies = [ "txl ==0.3.3", "txl_kernel", "pyzmq >=27.1.0", "ipykernel",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
12 changes: 6 additions & 6 deletions plugins/local_kernels/txl_local_kernels/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import uuid
from typing import Any, Dict, List, Optional, cast

from anyioutils import Task, create_task
from anyio import TaskHandle
from txl_kernel.driver import KernelMixin

from .connect import cfg_t, connect_channel, launch_kernel, read_connection_file
Expand Down Expand Up @@ -39,7 +39,7 @@ def __init__(
self.connection_cfg = read_connection_file(connection_file)
self.key = cast(str, self.connection_cfg["key"])
self.session_id = uuid.uuid4().hex
self.channel_tasks: List[Task] = []
self.channel_tasks: List[TaskHandle] = []
self.comm_handlers = comm_handlers
task_group.start_soon(self.start)
kernel_drivers.append(self)
Expand Down Expand Up @@ -83,8 +83,8 @@ def connect_channels(self, connection_cfg: Optional[cfg_t] = None):
self.iopub_channel = connect_channel("iopub", connection_cfg)

def listen_channels(self):
self.channel_tasks.append(create_task(self._recv_iopub(), self.task_group))
self.channel_tasks.append(create_task(self._recv_shell(), self.task_group))
self.channel_tasks.append(self.task_group.create_task(self._recv_iopub()))
self.channel_tasks.append(self.task_group.create_task(self._recv_shell()))

async def stop(self) -> None:
self.kernel_process.kill()
Expand All @@ -97,13 +97,13 @@ async def _recv_iopub(self):
while True:
msg = await self.receive_message(self.iopub_channel, change_str_to_date=True)
msg["channel"] = "iopub"
self.recv_queue.put_nowait(msg)
self.recv_queue.send_nowait(msg)

async def _recv_shell(self):
while True:
msg = await self.receive_message(self.shell_channel, change_str_to_date=True)
msg["channel"] = "shell"
self.recv_queue.put_nowait(msg)
self.recv_queue.send_nowait(msg)

async def send_message(
self,
Expand Down
2 changes: 1 addition & 1 deletion plugins/local_terminals/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
keywords = []
classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy",]
dependencies = [ "txl ==0.3.3", "pywinpty;platform_system=='Windows'", "anyio >=3.7.0,<5",]
dependencies = [ "txl ==0.3.3", "pywinpty;platform_system=='Windows'", "anyio >=4.14.2,<5",]
[[project.authors]]
name = "David Brochart"
email = "david.brochart@gmail.com"
Expand Down
24 changes: 15 additions & 9 deletions plugins/local_terminals/txl_local_terminals/terminal.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
import fcntl
import math
import os
import pty
import shlex
import struct
import termios
from typing import Any

from anyio import wait_readable
from anyioutils import Event, Queue
from anyio import Event, create_memory_object_stream, wait_readable
from textual.widget import Widget
from textual.widgets._header import HeaderTitle

from txl.base import Header, TerminalFactory, Terminals
from txl.stapled import StapledObjectStream


class TerminalsMeta(type(Terminals), type(Widget)):
Expand All @@ -22,8 +24,12 @@ def __init__(self, task_group, header: Header, terminal: TerminalFactory):
self.task_group = task_group
self.header = header
self.terminal = terminal
self._send_queue = Queue()
self._recv_queue = Queue()
self._send_queue = StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
)
self._recv_queue = StapledObjectStream(
*create_memory_object_stream[Any](max_buffer_size=math.inf)
)
self._data_or_disconnect = None
self._event = Event()
super().__init__()
Expand Down Expand Up @@ -66,9 +72,9 @@ async def _receive(self):
self._event.set()

async def _run(self):
await self._send_queue.put(["setup", {}])
await self._send_queue.send(["setup", {}])
while True:
msg = await self._recv_queue.get()
msg = await self._recv_queue.receive()
if msg[0] == "stdin" and msg[1] is not None:
self._p_out.write(msg[1].encode())
elif msg[0] == "set_size":
Expand All @@ -78,8 +84,8 @@ async def _run(self):
async def _send(self):
while True:
await self._event.wait()
self._event.clear()
self._event = Event()
if self._data_or_disconnect is None:
await self._send_queue.put(["disconnect", 1])
await self._send_queue.send(["disconnect", 1])
else:
await self._send_queue.put(["stdout", self._data_or_disconnect])
await self._send_queue.send(["stdout", self._data_or_disconnect])
Loading
Loading