feat(kvcache): checkpoint decode state at the tool-call opener token

2026-08-13 18:32:02 +00:00
parent ee54f638fd
commit ea5348b4e3
10 changed files with 160 additions and 8 deletions
@@ -59,6 +59,7 @@ def _setup():
        prefill_manager=pm,
        finished_reqs=set(),
        eos_token_ids=set(),
+        toolcall_anchor_id=None,
        config=SimpleNamespace(page_size=1),
        status_reporter=SimpleNamespace(report_batch=lambda *_, **__: None),
        send_result=sent.extend,