diff --git a/src/ai/unified-ai-runner.openai.ts b/src/ai/unified-ai-runner.openai.ts index 68b37bd..c4d058f 100644 --- a/src/ai/unified-ai-runner.openai.ts +++ b/src/ai/unified-ai-runner.openai.ts @@ -5,14 +5,53 @@ import {getOpenAITools} from "./tool-mappers"; import {TelegramStreamMessage} from "./telegram-stream-message"; import {ToolRuntimeContext} from "./tools/runtime"; import {OpenAIChatMessage} from "./openai-chat-message"; -import type {ResponseCreateParamsNonStreaming, ResponseCreateParamsStreaming, ResponseInputItem, ResponseStreamEvent} from "openai/resources/responses/responses"; -import type {ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsStreaming} from "openai/resources/chat/completions"; +import type { + ResponseCreateParamsNonStreaming, + ResponseCreateParamsStreaming, + ResponseInputItem, + ResponseStreamEvent +} from "openai/resources/responses/responses"; +import type { + ChatCompletionCreateParamsNonStreaming, + ChatCompletionCreateParamsStreaming +} from "openai/resources/chat/completions"; import {createGeminiOpenAiClient, createOpenAiClient} from "./ai-runtime-target"; import {aiLog, aiLogDuration, aiLogMessageIdentity, aiLogProviderTarget, aiLogToolCall} from "../logging/ai-logger"; -import {AsyncIterableStream, MAX_TOOL_ROUNDS, OPENAI_IMAGE_PARTIALS, OpenAiChatCompletionResponseLike, OpenAiChatToolCallLike, OpenAiCompatibleChatMessage, OpenAiCompatibleContentPart, OpenAiResponseLike, OpenAiResponseOutputItem, RuntimeConfigSnapshot, ToolCallData, StreamingToolCallAccumulator, collectOpenAiResponseFunctionCalls, collectOpenAiResponseImages, collectOpenAiResponseText, executeToolBatch, getOpenAIResponsesToolsWithImage, openAiResponseItemCallId, safeJsonParseObject, showOpenAiGeneratedImage, ToolExecutionMemory, isRecord, roundStatus, OpenAiChatCompletionStreamChunkLike} from "./unified-ai-runner.shared"; +import { + AsyncIterableStream, + collectOpenAiResponseFunctionCalls, + collectOpenAiResponseImages, + collectOpenAiResponseText, + executeToolBatch, + getOpenAIResponsesToolsWithImage, + isRecord, + MAX_TOOL_ROUNDS, + OPENAI_IMAGE_PARTIALS, + OpenAiChatCompletionResponseLike, + OpenAiChatCompletionStreamChunkLike, + OpenAiChatToolCallLike, + OpenAiCompatibleChatMessage, + OpenAiCompatibleContentPart, + openAiResponseItemCallId, + OpenAiResponseLike, + OpenAiResponseOutputItem, + roundStatus, + RuntimeConfigSnapshot, + safeJsonParseObject, + showOpenAiGeneratedImage, + StreamingToolCallAccumulator, + ToolCallData, + ToolExecutionMemory +} from "./unified-ai-runner.shared"; +import {GetNoteFileResult, GetNoteFileResultSchema} from "./tools/send-note-file"; +import {bot, notesDir} from "../index"; +import fs from "node:fs"; +import path from "node:path"; +import {logError} from "../util/utils"; export async function runOpenAi( + msg: Message, messages: OpenAIChatMessage[], streamMessage: TelegramStreamMessage, signal: AbortSignal, @@ -90,6 +129,32 @@ export async function runOpenAi( argumentsText: call.argumentsText, })); const toolResults = await executeToolBatch(toolCalls, streamMessage, toolContext, toolMemory); + + let successGetNoteFileResult: GetNoteFileResult | undefined = undefined; + + for (const toolResult of toolResults) { + try { + const raw = JSON.parse(toolResult); + const res = GetNoteFileResultSchema.safeParse(raw); + + if (res.success && res.data.success) { + successGetNoteFileResult = res.data; + } + } catch { + // Not every tool result is JSON. + } + } + + if (successGetNoteFileResult && "attachment" in successGetNoteFileResult) { + await bot.sendDocument({ + chat_id: msg.chat.id, + reply_parameters: { + message_id: msg.message_id, + }, + document: fs.createReadStream(path.join(notesDir, successGetNoteFileResult.attachment.relativePath)), + }).catch(logError); + } + const toolOutputs = calls.map((call, index) => ({ type: "function_call_output" as const, call_id: call.callId, @@ -228,6 +293,32 @@ export async function runOpenAi( argumentsText: call.argumentsText, })); const toolResults = await executeToolBatch(toolCalls, streamMessage, toolContext, toolMemory); + + let successGetNoteFileResult: GetNoteFileResult | undefined = undefined; + + for (const toolResult of toolResults) { + try { + const raw = JSON.parse(toolResult); + const res = GetNoteFileResultSchema.safeParse(raw); + + if (res.success && res.data.success) { + successGetNoteFileResult = res.data; + } + } catch { + // Not every tool result is JSON. + } + } + + if (successGetNoteFileResult && "attachment" in successGetNoteFileResult) { + await bot.sendDocument({ + chat_id: msg.chat.id, + reply_parameters: { + message_id: msg.message_id, + }, + document: fs.createReadStream(path.join(notesDir, successGetNoteFileResult.attachment.relativePath)), + }).catch(logError); + } + const toolOutputs = calls.map((call, index) => ({ type: "function_call_output", call_id: call.callId, @@ -298,6 +389,7 @@ async function appendOpenAiChatToolResults( } export async function runOpenAiCompatibleChat( + msg: Message, messages: OpenAIChatMessage[], streamMessage: TelegramStreamMessage, signal: AbortSignal, @@ -356,7 +448,35 @@ export async function runOpenAiCompatibleChat( }, })), }); - await appendOpenAiChatToolResults(chatMessages, calls, await executeToolBatch(calls, streamMessage, toolContext, toolMemory)); + + const toolResults = await executeToolBatch(calls, streamMessage, toolContext, toolMemory); + + let successGetNoteFileResult: GetNoteFileResult | undefined = undefined; + + for (const toolResult of toolResults) { + try { + const raw = JSON.parse(toolResult); + const res = GetNoteFileResultSchema.safeParse(raw); + + if (res.success && res.data.success) { + successGetNoteFileResult = res.data; + } + } catch { + // Not every tool result is JSON. + } + } + + if (successGetNoteFileResult && "attachment" in successGetNoteFileResult) { + await bot.sendDocument({ + chat_id: msg.chat.id, + reply_parameters: { + message_id: msg.message_id, + }, + document: fs.createReadStream(path.join(notesDir, successGetNoteFileResult.attachment.relativePath)), + }).catch(logError); + } + + await appendOpenAiChatToolResults(chatMessages, calls, toolResults); continue; } @@ -410,15 +530,34 @@ export async function runOpenAiCompatibleChat( }, })), }); - await appendOpenAiChatToolResults(chatMessages, calls, await executeToolBatch(calls, streamMessage, toolContext, toolMemory)); + + const toolResults = await executeToolBatch(calls, streamMessage, toolContext, toolMemory); + + let successGetNoteFileResult: GetNoteFileResult | undefined = undefined; + + for (const toolResult of toolResults) { + try { + const raw = JSON.parse(toolResult); + const res = GetNoteFileResultSchema.safeParse(raw); + + if (res.success && res.data.success) { + successGetNoteFileResult = res.data; + } + } catch { + // Not every tool result is JSON. + } + } + + if (successGetNoteFileResult && "attachment" in successGetNoteFileResult) { + await bot.sendDocument({ + chat_id: msg.chat.id, + reply_parameters: { + message_id: msg.message_id, + }, + document: fs.createReadStream(path.join(notesDir, successGetNoteFileResult.attachment.relativePath)), + }).catch(logError); + } + + await appendOpenAiChatToolResults(chatMessages, calls, toolResults); } } - - -export class OpenAiProviderRunner { - static run = runOpenAi; -} - -export class OpenAiCompatibleProviderRunner { - static run = runOpenAiCompatibleChat; -} diff --git a/src/ai/unified-ai-runner.shared.ts b/src/ai/unified-ai-runner.shared.ts index 10fb31c..7af3ebe 100644 --- a/src/ai/unified-ai-runner.shared.ts +++ b/src/ai/unified-ai-runner.shared.ts @@ -27,7 +27,7 @@ import { transcribeSpeechDownloads } from "./speech-to-text"; import {OpenAIChatMessage} from "./openai-chat-message"; -import type {ResponseInputMessageContentList} from "openai/resources/responses/responses"; +import type {ResponseInputContent, ResponseInputMessageContentList} from "openai/resources/responses/responses"; import type {ChatCompletionMessageParam} from "openai/resources/chat/completions"; import type {GenerateContentParameters} from "@google/genai"; import {MistralChatMessage} from "./mistral-chat-message"; @@ -536,13 +536,13 @@ export function addMessageAttachmentKinds(msg: Message | undefined, kinds: Set { +export async function collectStoredReplyChainAttachments(msg: Message, limit: number = 1): Promise { const attachments: StoredAttachment[] = []; const seen = new Set(); let current = await MessageStore.get(msg.chat.id, msg.message_id); - for (let i = 0; current && i < 40; i++) { - for (const attachment of current.attachments ?? []) { + for (let i = 0; current && i < limit; i++) { + for (const attachment of current?.attachments ?? []) { const key = [ attachment.kind, attachment.fileUniqueId || attachment.fileId, @@ -796,8 +796,8 @@ export function normalizeOllamaToolCalls(calls: readonly OllamaToolCallLike[] = } export function buildOpenAiResponseMessage(part: MessagePart, getContent: (part: MessagePart) => string): OpenAIChatMessage { - const content: ResponseInputMessageContentList = [{ - type: "input_text", + const content: Array = [{ + type: part.bot ? "output_text" : "input_text", text: getContent(part), }]; diff --git a/src/ai/unified-ai-runner.ts b/src/ai/unified-ai-runner.ts index 80a8aea..10882b2 100644 --- a/src/ai/unified-ai-runner.ts +++ b/src/ai/unified-ai-runner.ts @@ -137,7 +137,7 @@ async function executeUnifiedAiRequest( switch (options.provider) { case AiProvider.OPENAI: - await runOpenAi(chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, options.msg, config, toolContext); + await runOpenAi(options.msg, chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, options.msg, config, toolContext); break; case AiProvider.OLLAMA: const currentModel = config.ollamaChatTarget.model; @@ -154,7 +154,7 @@ async function executeUnifiedAiRequest( break; case AiProvider.GEMINI: if (getGeminiApiMode(config.geminiChatTarget) === "openai") { - await runOpenAiCompatibleChat(chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, config, toolContext); + await runOpenAiCompatibleChat(options.msg, chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, config, toolContext); } else { await runGemini(chatMessages as GeminiMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, config, toolContext); }