|
| 1 | +"""The tiktoken loader must not leave a daemon thread inside an import. |
| 2 | +
|
| 3 | +``import tiktoken`` dlopens a Rust extension. A daemon thread killed |
| 4 | +mid-import at interpreter finalization takes glibc's ``_dl_load_lock`` |
| 5 | +with it (later SIGSEGV) or unwinds through an ``extern "C"`` frame |
| 6 | +(``abort()``); both were observed in production as a signal death |
| 7 | +*after* a fully correct protocol stream. So the import belongs on the |
| 8 | +caller thread and only ``get_encoding`` on the daemon thread — these |
| 9 | +tests pin which thread runs which, and that the exit hook drains the |
| 10 | +load. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import importlib.util |
| 16 | +import sys |
| 17 | +import threading |
| 18 | +import types |
| 19 | +from typing import Any |
| 20 | + |
| 21 | +import pytest |
| 22 | + |
| 23 | +from agent_core.runtime.loop import tokenizer |
| 24 | + |
| 25 | + |
| 26 | +class _RecordingLoader: |
| 27 | + """Stands in for the real ``tiktoken``, recording the loading thread.""" |
| 28 | + |
| 29 | + def __init__(self, gate: threading.Event | None = None) -> None: |
| 30 | + self.import_thread: str | None = None |
| 31 | + self.encode_thread: str | None = None |
| 32 | + self.encoder = object() |
| 33 | + self._gate = gate |
| 34 | + |
| 35 | + # -- meta_path finder ------------------------------------------------- |
| 36 | + def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> Any: |
| 37 | + if fullname != "tiktoken": |
| 38 | + return None |
| 39 | + return importlib.util.spec_from_loader(fullname, self) |
| 40 | + |
| 41 | + def create_module(self, spec: Any) -> types.ModuleType: |
| 42 | + return types.ModuleType(spec.name) |
| 43 | + |
| 44 | + def exec_module(self, module: types.ModuleType) -> None: |
| 45 | + self.import_thread = threading.current_thread().name |
| 46 | + module.get_encoding = self._get_encoding # pyright: ignore[reportAttributeAccessIssue] |
| 47 | + |
| 48 | + # -- the fake tiktoken API ------------------------------------------- |
| 49 | + def _get_encoding(self, name: str) -> object: |
| 50 | + self.encode_thread = threading.current_thread().name |
| 51 | + if self._gate is not None: |
| 52 | + self._gate.wait(timeout=5.0) |
| 53 | + return self.encoder |
| 54 | + |
| 55 | + |
| 56 | +@pytest.fixture |
| 57 | +def clean_tokenizer(): |
| 58 | + """Reset the module cache and unhook any real/fake tiktoken.""" |
| 59 | + saved_module = sys.modules.pop("tiktoken", None) |
| 60 | + saved_meta = list(sys.meta_path) |
| 61 | + saved_atexit_registered = tokenizer._atexit_registered |
| 62 | + tokenizer._encoders.clear() |
| 63 | + tokenizer._threads.clear() |
| 64 | + tokenizer._atexit_registered = True # don't leak a real atexit hook per test |
| 65 | + try: |
| 66 | + yield |
| 67 | + finally: |
| 68 | + tokenizer._join_pending() |
| 69 | + tokenizer._encoders.clear() |
| 70 | + tokenizer._threads.clear() |
| 71 | + tokenizer._atexit_registered = saved_atexit_registered |
| 72 | + sys.meta_path[:] = saved_meta |
| 73 | + sys.modules.pop("tiktoken", None) |
| 74 | + if saved_module is not None: |
| 75 | + sys.modules["tiktoken"] = saved_module |
| 76 | + |
| 77 | + |
| 78 | +def test_import_runs_on_caller_thread_get_encoding_on_daemon(clean_tokenizer) -> None: |
| 79 | + loader = _RecordingLoader() |
| 80 | + sys.meta_path.insert(0, loader) |
| 81 | + |
| 82 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is None |
| 83 | + # The import must already have happened, synchronously, right here. |
| 84 | + assert loader.import_thread == threading.current_thread().name |
| 85 | + assert "tiktoken" in sys.modules |
| 86 | + |
| 87 | + tokenizer._join_pending() |
| 88 | + assert loader.encode_thread == "tiktoken-init-cl100k_base" |
| 89 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is loader.encoder |
| 90 | + |
| 91 | + |
| 92 | +def test_atexit_hook_registers_once_and_drains_the_in_flight_load( |
| 93 | + clean_tokenizer, monkeypatch: pytest.MonkeyPatch |
| 94 | +) -> None: |
| 95 | + gate = threading.Event() |
| 96 | + loader = _RecordingLoader(gate=gate) |
| 97 | + sys.meta_path.insert(0, loader) |
| 98 | + registered: list[Any] = [] |
| 99 | + monkeypatch.setattr(tokenizer.atexit, "register", registered.append) |
| 100 | + tokenizer._atexit_registered = False |
| 101 | + |
| 102 | + tokenizer.get_encoding_nonblocking("cl100k_base") |
| 103 | + tokenizer.get_encoding_nonblocking("o200k_base") |
| 104 | + assert "cl100k_base" in tokenizer._threads # in flight |
| 105 | + assert registered == [tokenizer._join_pending] |
| 106 | + |
| 107 | + gate.set() |
| 108 | + registered[0]() |
| 109 | + assert tokenizer._threads == {} # thread finished and deregistered itself |
| 110 | + assert tokenizer._encoders["cl100k_base"] is loader.encoder |
| 111 | + |
| 112 | + |
| 113 | +def test_atexit_join_timeout_is_shared_across_threads( |
| 114 | + clean_tokenizer, monkeypatch: pytest.MonkeyPatch |
| 115 | +) -> None: |
| 116 | + class _Clock: |
| 117 | + now = 10.0 |
| 118 | + |
| 119 | + def monotonic(self) -> float: |
| 120 | + return self.now |
| 121 | + |
| 122 | + class _PendingThread: |
| 123 | + def __init__(self, clock: _Clock) -> None: |
| 124 | + self.clock = clock |
| 125 | + self.timeouts: list[float] = [] |
| 126 | + |
| 127 | + def join(self, timeout: float | None = None) -> None: |
| 128 | + assert timeout is not None |
| 129 | + self.timeouts.append(timeout) |
| 130 | + self.clock.now += 0.4 |
| 131 | + |
| 132 | + clock = _Clock() |
| 133 | + pending = [_PendingThread(clock) for _ in range(4)] |
| 134 | + tokenizer._threads.update({str(i): thread for i, thread in enumerate(pending)}) # type: ignore[arg-type] |
| 135 | + monkeypatch.setattr(tokenizer.time, "monotonic", clock.monotonic) |
| 136 | + |
| 137 | + tokenizer._join_pending() |
| 138 | + |
| 139 | + assert [thread.timeouts for thread in pending] == [ |
| 140 | + [pytest.approx(1.0)], |
| 141 | + [pytest.approx(0.6)], |
| 142 | + [pytest.approx(0.2)], |
| 143 | + [], |
| 144 | + ] |
| 145 | + |
| 146 | + |
| 147 | +def test_caller_never_blocks_on_a_wedged_get_encoding(clean_tokenizer) -> None: |
| 148 | + gate = threading.Event() |
| 149 | + loader = _RecordingLoader(gate=gate) |
| 150 | + sys.meta_path.insert(0, loader) |
| 151 | + try: |
| 152 | + # Returns immediately even though get_encoding is parked. |
| 153 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is None |
| 154 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is None |
| 155 | + finally: |
| 156 | + gate.set() |
| 157 | + |
| 158 | + |
| 159 | +def test_missing_tiktoken_is_terminal_and_spawns_no_thread(clean_tokenizer) -> None: |
| 160 | + class _Blocker: |
| 161 | + def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> Any: |
| 162 | + if fullname == "tiktoken": |
| 163 | + raise ImportError("no tiktoken") |
| 164 | + return None |
| 165 | + |
| 166 | + sys.meta_path.insert(0, _Blocker()) |
| 167 | + |
| 168 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is None |
| 169 | + assert tokenizer._threads == {} |
| 170 | + assert tokenizer._encoders["cl100k_base"] is False |
| 171 | + assert tokenizer.get_encoding_nonblocking("cl100k_base") is None |
| 172 | + |
| 173 | + |
| 174 | +def test_concurrent_callers_spawn_one_thread(clean_tokenizer) -> None: |
| 175 | + gate = threading.Event() |
| 176 | + loader = _RecordingLoader(gate=gate) |
| 177 | + sys.meta_path.insert(0, loader) |
| 178 | + try: |
| 179 | + start = threading.Barrier(4) |
| 180 | + |
| 181 | + def call() -> None: |
| 182 | + start.wait(timeout=5.0) |
| 183 | + tokenizer.get_encoding_nonblocking("cl100k_base") |
| 184 | + |
| 185 | + callers = [threading.Thread(target=call) for _ in range(4)] |
| 186 | + for t in callers: |
| 187 | + t.start() |
| 188 | + for t in callers: |
| 189 | + t.join(timeout=5.0) |
| 190 | + |
| 191 | + assert len(tokenizer._threads) == 1 |
| 192 | + finally: |
| 193 | + gate.set() |
0 commit comments