fix(chat): use native async LLM call to prevent session reset on message submit
The previous implementation used asyncio.to_thread(llm.complete) to avoid blocking the event loop, but Chainlit's contextvars context is not reliably propagated across thread boundaries, causing the session to reset and clear the message history on each user submission. Changes: - Add llm.acomplete() using litellm.acompletion() (native coroutine) - Replace asyncio.to_thread() in on_message with await llm.acomplete() - Store the welcome message in state.messages so it is replayed on reconnect - Persist state to cl.user_session immediately after appending the user's message (before the LLM call) so reconnect detection has the latest history - Add pytest-asyncio dev dependency and asyncio_mode = "auto" config - Add 6 async tests for acomplete() in tests/test_llm.py https://claude.ai/code/session_01SUWzMzFvSfWiHXA2p6rPg9
This commit is contained in:
+21
@@ -5,6 +5,27 @@ from collections.abc import Generator
|
||||
import litellm
|
||||
|
||||
|
||||
async def acomplete(model: str, system: str, messages: list) -> str:
|
||||
"""Call any LLM asynchronously and return the response text.
|
||||
|
||||
This is the async equivalent of ``complete()`` — use this from async
|
||||
handlers (e.g. Chainlit's ``@cl.on_message``) to avoid blocking the
|
||||
event loop and losing framework context variables.
|
||||
|
||||
Args:
|
||||
model: litellm model string, e.g. "anthropic/claude-opus-4-6".
|
||||
system: System prompt text.
|
||||
messages: List of objects with .role and .content attributes.
|
||||
|
||||
Returns:
|
||||
The model's reply as a plain string.
|
||||
"""
|
||||
api_messages = [{"role": "system", "content": system}]
|
||||
api_messages += [{"role": m.role, "content": m.content} for m in messages]
|
||||
response = await litellm.acompletion(model=model, messages=api_messages, max_tokens=2048)
|
||||
return response.choices[0].message.content
|
||||
|
||||
|
||||
def complete(model: str, system: str, messages: list) -> str:
|
||||
"""Call any LLM and return the response text.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user