diff --git a/tools/server/webui/public/demo-conversation.json b/tools/server/webui/public/demo-conversation.json index 338b4aea590f2..daa1e97a04ad2 100644 --- a/tools/server/webui/public/demo-conversation.json +++ b/tools/server/webui/public/demo-conversation.json @@ -28,6 +28,36 @@ "id": 1734087548329, "role": "assistant", "content": "Code block:\n```js\nconsole.log('hello world')\n```\n```sh\nls -la /dev\n```" + }, + { + "id": 1734087548330, + "role": "user", + "content": "What is the weather in San Francisco?" + }, + { + "id": 1734087548331, + "role": "assistant", + "content": "User wants to know what's te weather like in San Francisco.Let me check that for you!", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"San Francisco\"\n}" + } + } + ] + }, + { + "id": 1734087548332, + "role": "tool", + "content": "{\"temperature\": 20, \"condition\": \"sunny\"}" + }, + { + "id": 1734087548333, + "role": "assistant", + "content": "The weather in San Francisco is currently 20°C and sunny." } ] } diff --git a/tools/server/webui/src/Config.ts b/tools/server/webui/src/Config.ts index c03ac287f3484..707702028fe7c 100644 --- a/tools/server/webui/src/Config.ts +++ b/tools/server/webui/src/Config.ts @@ -41,6 +41,7 @@ export const CONFIG_DEFAULT = { custom: '', // custom json-stringified object // experimental features pyIntepreterEnabled: false, + toolJsReplEnabled: false, }; export const CONFIG_INFO: Record = { apiKey: 'Set the API Key if you are using --api-key option for the server.', diff --git a/tools/server/webui/src/assets/iframe_sandbox.html b/tools/server/webui/src/assets/iframe_sandbox.html new file mode 100644 index 0000000000000..99cd55f5eca0f --- /dev/null +++ b/tools/server/webui/src/assets/iframe_sandbox.html @@ -0,0 +1,77 @@ + + + + JS Sandbox + + + +

JavaScript Execution Sandbox

+ + diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index ee59de450d1ff..e914c8833cf98 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -1,8 +1,10 @@ import { useMemo, useState } from 'react'; import { useAppContext } from '../utils/app.context'; import { Message, PendingMessage } from '../utils/types'; -import { classNames } from '../utils/misc'; +import { classNames, parseThoughtContent } from '../utils/misc'; import MarkdownDisplay, { CopyButton } from './MarkdownDisplay'; +import { ToolCallArgsDisplay } from './tool_calling/ToolCallArgsDisplay'; +import { ToolCallResultDisplay } from './tool_calling/ToolCallResultDisplay'; import { ArrowPathIcon, ChevronLeftIcon, @@ -18,8 +20,87 @@ interface SplitMessage { isThinking?: boolean; } +// Helper function to extract thoughts using shared parseThoughtContent +function extractThoughts(content: string | null, role: string): SplitMessage { + if (content === null || (role !== 'assistant' && role !== 'tool')) { + return { content }; + } + + const { filteredContent, thoughtContent, isThinking } = + parseThoughtContent(content); + + return { + content: filteredContent, + thought: thoughtContent, + isThinking, + }; +} + +// Helper component to render a single message part +function MessagePart({ + message, + isPending, + showThoughts = true, + className = '', + baseClassName = '', + isMainMessage = false, +}: { + message: Message | PendingMessage; + isPending?: boolean; + showThoughts?: boolean; + className?: string; + baseClassName?: string; + isMainMessage?: boolean; +}) { + const { config } = useAppContext(); + const { content, thought, isThinking } = extractThoughts( + message.content, + message.role + ); + + if (message.role === 'tool' && baseClassName) { + return ( + + ); + } + + return ( +
+ {showThoughts && thought && ( + + )} + + {message.role === 'tool' && content ? ( + + ) : ( + content && + content.trim() !== '' && ( + + ) + )} + + {message.tool_calls && + message.tool_calls.map((toolCall) => ( + + ))} +
+ ); +} + export default function ChatMessage({ msg, + chainedParts, siblingLeafNodeIds, siblingCurrIdx, id, @@ -29,6 +110,7 @@ export default function ChatMessage({ isPending, }: { msg: Message | PendingMessage; + chainedParts?: (Message | PendingMessage)[]; siblingLeafNodeIds: Message['id'][]; siblingCurrIdx: number; id?: string; @@ -55,34 +137,23 @@ export default function ChatMessage({ const nextSibling = siblingLeafNodeIds[siblingCurrIdx + 1]; const prevSibling = siblingLeafNodeIds[siblingCurrIdx - 1]; - // for reasoning model, we split the message into content and thought - // TODO: implement this as remark/rehype plugin in the future - const { content, thought, isThinking }: SplitMessage = useMemo(() => { - if (msg.content === null || msg.role !== 'assistant') { - return { content: msg.content }; - } - let actualContent = ''; - let thought = ''; - let isThinking = false; - let thinkSplit = msg.content.split('', 2); - actualContent += thinkSplit[0]; - while (thinkSplit[1] !== undefined) { - // tag found - thinkSplit = thinkSplit[1].split('', 2); - thought += thinkSplit[0]; - isThinking = true; - if (thinkSplit[1] !== undefined) { - // closing tag found - isThinking = false; - thinkSplit = thinkSplit[1].split('', 2); - actualContent += thinkSplit[0]; - } - } - return { content: actualContent, thought, isThinking }; - }, [msg]); + const mainSplitMessage = useMemo( + () => extractThoughts(msg.content, msg.role), + [msg.content, msg.role] + ); if (!viewingChat) return null; + const toolCalls = msg.tool_calls ?? null; + + const hasContentInMainMsg = + mainSplitMessage.content && mainSplitMessage.content.trim() !== ''; + const hasContentInChainedParts = chainedParts?.some((part) => { + const splitPart = extractThoughts(part.content, part.role); + return splitPart.content && splitPart.content.trim() !== ''; + }); + const entireTurnHasSomeDisplayableContent = + hasContentInMainMsg || hasContentInChainedParts; const isUser = msg.role === 'user'; return ( @@ -141,28 +212,40 @@ export default function ChatMessage({ {/* not editing content, render message */} {editingContent === null && ( <> - {content === null ? ( + {mainSplitMessage.content === null && + !toolCalls && + !chainedParts?.length ? ( <> {/* show loading dots for pending message */} ) : ( <> - {/* render message as markdown */} + {/* render main message */}
- {thought && ( - - )} - -
+ + {/* render chained parts */} + {chainedParts?.map((part) => ( + + ))} )} {/* render timings if enabled */} @@ -195,7 +278,7 @@ export default function ChatMessage({ {/* actions for each message */} - {msg.content !== null && ( + {(entireTurnHasSomeDisplayableContent || msg.role === 'user') && (
{ - if (msg.content !== null) { + if (entireTurnHasSomeDisplayableContent) { onRegenerateMessage(msg as Message); } }} - disabled={msg.content === null} + disabled={!entireTurnHasSomeDisplayableContent} tooltipsContent="Regenerate response" > @@ -263,7 +346,23 @@ export default function ChatMessage({ )} )} - + {entireTurnHasSomeDisplayableContent && ( + p.content) + .map((p) => { + if (p.role === 'user') { + return p.content; + } else { + return extractThoughts(p.content, p.role).content; + } + }) + .join('\n\n') || '' + } + /> + )}
)} diff --git a/tools/server/webui/src/components/ChatScreen.tsx b/tools/server/webui/src/components/ChatScreen.tsx index 09c601ef2366a..fc339ed8d5aff 100644 --- a/tools/server/webui/src/components/ChatScreen.tsx +++ b/tools/server/webui/src/components/ChatScreen.tsx @@ -27,6 +27,7 @@ import { scrollToBottom, useChatScroll } from './useChatScroll.tsx'; */ export interface MessageDisplay { msg: Message | PendingMessage; + chainedParts?: (Message | PendingMessage)[]; // For merging consecutive assistant/tool messages siblingLeafNodeIds: Message['id'][]; siblingCurrIdx: number; isPending?: boolean; @@ -69,18 +70,72 @@ function getListMessageDisplay( } return currNode?.id ?? -1; }; + const processedIds = new Set(); // traverse the current nodes - for (const msg of currNodes) { - const parentNode = nodeMap.get(msg.parent ?? -1); - if (!parentNode) continue; - const siblings = parentNode.children; - if (msg.type !== 'root') { - res.push({ - msg, - siblingLeafNodeIds: siblings.map(findLeafNode), - siblingCurrIdx: siblings.indexOf(msg.id), - }); + for (const currentMessage of currNodes) { + if (processedIds.has(currentMessage.id) || currentMessage.type === 'root') { + continue; } + + const displayMsg = currentMessage; + const chainedParts: (Message | PendingMessage)[] = []; + processedIds.add(displayMsg.id); + + if (displayMsg.role === 'assistant') { + let currentLinkInChain = displayMsg; // Start with the initial assistant message + + // Loop to chain subsequent tool calls and their assistant responses + while (true) { + if (currentLinkInChain.children.length !== 1) { + // Stop if there isn't a single, clear next step in the chain + // or if the current link has no children. + break; + } + + const childId = currentLinkInChain.children[0]; + const childNode = nodeMap.get(childId); + + if (!childNode || processedIds.has(childNode.id)) { + // Child not found or already processed, end of chain + break; + } + + // Scenario 1: Current is Assistant, next is Tool + if ( + currentLinkInChain.role === 'assistant' && + childNode.role === 'tool' + ) { + chainedParts.push(childNode); + processedIds.add(childNode.id); + currentLinkInChain = childNode; // Continue chain from the tool message + } + // Scenario 2: Current is Tool, next is Assistant + else if ( + currentLinkInChain.role === 'tool' && + childNode.role === 'assistant' + ) { + chainedParts.push(childNode); + processedIds.add(childNode.id); + currentLinkInChain = childNode; // Continue chain from the assistant message + // This assistant message might make further tool calls + } + // Scenario 3: Pattern broken (e.g., Assistant -> Assistant, or Tool -> Tool) + else { + break; // Pattern broken, end of this specific tool-use chain + } + } + } + + const parentNode = nodeMap.get(displayMsg.parent ?? -1); + if (!parentNode && displayMsg.type !== 'root') continue; // Skip if parent not found for non-root + + const siblings = parentNode ? parentNode.children : []; + res.push({ + msg: displayMsg, + chainedParts: chainedParts.length > 0 ? chainedParts : undefined, + siblingLeafNodeIds: siblings.map(findLeafNode), + siblingCurrIdx: siblings.indexOf(displayMsg.id), + }); } return res; } @@ -136,13 +191,33 @@ export default function ChatScreen() { } textarea.setValue(''); scrollToBottom(false); + + // Determine the ID of the actual last message to use as parent + let parentMessageId: Message['id'] | null = null; + const lastMessageDisplayItem = messages.at(-1); + + if (lastMessageDisplayItem) { + if ( + lastMessageDisplayItem.chainedParts && + lastMessageDisplayItem.chainedParts.length > 0 + ) { + // If the last display item has chained parts, the true last message is the last part of that chain + parentMessageId = + lastMessageDisplayItem.chainedParts.at(-1)?.id ?? + lastMessageDisplayItem.msg.id; + } else { + // Otherwise, it's the main message of the last display item + parentMessageId = lastMessageDisplayItem.msg.id; + } + } + // If messages is empty (e.g., new chat), parentMessageId will remain null. + // sendMessage handles parentId = null correctly for starting new conversations. setCurrNodeId(-1); - // get the last message node - const lastMsgNodeId = messages.at(-1)?.msg.id ?? null; + if ( !(await sendMessage( currConvId, - lastMsgNodeId, + parentMessageId, lastInpMsg, extraContext.items, onChunk @@ -248,6 +323,7 @@ export default function ChatScreen() { ; - label: string | React.ReactElement; - help?: string | React.ReactElement; + label: string | ReactElement; + help?: string | ReactElement; key: SettKey; } @@ -169,6 +170,31 @@ const SETTING_SECTIONS: SettingSection[] = [ }, ], }, + { + title: ( + <> + + Tool Calling + + ), + fields: [ + { + type: SettingInputType.CHECKBOX, + label: ( + <> + JavaScript Interpreter + + Agent tool description: + Executes JavaScript code in a sandboxed iframe. The code should be + self-contained valid javascript. Only console.log(variable) and + final result are included in response content. + + + ), + key: 'toolJsReplEnabled', + }, + ], + }, { title: ( <> @@ -184,10 +210,7 @@ const SETTING_SECTIONS: SettingSection[] = [ const debugImportDemoConv = async () => { const res = await fetch('/demo-conversation.json'); const demoConv = await res.json(); - StorageUtils.remove(demoConv.id); - for (const msg of demoConv.messages) { - StorageUtils.appendMsg(demoConv.id, msg); - } + await StorageUtils.importDemoConversation(demoConv); }; return (