From 067bbd07083c3c2550e3ddba9275d5057fe928e4 Mon Sep 17 00:00:00 2001 From: Danil Nikolaev Date: Thu, 14 May 2026 20:55:48 +0300 Subject: [PATCH] shitton --- bun.lock | 20 +- locales/en.json | 1 + locales/ru.json | 3 +- locales/ua.json | 3 +- package-lock.json | 135 ++-------- package.json | 2 +- src/ai/ai-runtime-target.ts | 1 + src/ai/provider-model-runtime.ts | 6 + src/ai/tool-mappers.ts | 33 ++- src/ai/tools/brave-search.ts | 4 +- src/ai/tools/list-notes.ts | 32 ++- src/ai/tools/market-rates.ts | 14 +- src/ai/tools/python-interpretator.ts | 17 +- src/ai/tools/registry.ts | 33 ++- ...send-note-file.ts => send-note-as-file.ts} | 6 +- src/ai/tools/shell.ts | 2 +- src/ai/tools/utils.ts | 25 +- src/ai/unified-ai-runner.ollama.ts | 167 +++++++----- src/ai/unified-ai-runner.openai.ts | 53 +++- src/ai/unified-ai-runner.shared.ts | 41 ++- src/ai/unified-ai-runner.tool-ranker.ts | 242 +++++++++--------- src/ai/unified-ai-runner.ts | 43 +++- src/commands/openai-chat.ts | 2 +- src/common/environment.ts | 2 + src/model/ai-model-capabilities.ts | 1 + 25 files changed, 496 insertions(+), 392 deletions(-) rename src/ai/tools/{send-note-file.ts => send-note-as-file.ts} (97%) diff --git a/bun.lock b/bun.lock index c338d01..7b571ea 100644 --- a/bun.lock +++ b/bun.lock @@ -28,8 +28,8 @@ "@types/fluent-ffmpeg": "^2.1.28", "@types/node": "^25.6.1", "@types/qrcode": "^1.5.6", - "@typescript/native-preview": "^7.0.0-beta", "drizzle-kit": "^0.31.10", + "typescript": "^6.0.3", }, }, }, @@ -236,22 +236,6 @@ "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], - "@typescript/native-preview": ["@typescript/native-preview@7.0.0-dev.20260421.2", "", { "optionalDependencies": { "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-arm": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-x64": "7.0.0-dev.20260421.2", "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-win32-x64": "7.0.0-dev.20260421.2" }, "bin": { "tsgo": "bin/tsgo.js" } }, "sha512-CmajHI25HpVWE9R1XFoxr+cphJPxoYD3eFioQtAvXYkMFKnLdICMS9pXre9Pybizb75ejRxjKD5/CVG055rEIg=="], - - "@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260421.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-fHv1r3ZmVo6zxuAIFmuX3w9QxbcauoG0SsWhmDwm6VmRubLlOJIcmTtlmV3JAb9oOnq8LuzZljzT7Q39fSMQDw=="], - - "@typescript/native-preview-darwin-x64": ["@typescript/native-preview-darwin-x64@7.0.0-dev.20260421.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-KWTR6xbW9t+JS7D5DQIzo75pqVXVWUxF9PMv/+S6xsnOjCVd6g0ixHcFpFMJMKSUQpGPr8Z5f7b8ks6LHW01jg=="], - - "@typescript/native-preview-linux-arm": ["@typescript/native-preview-linux-arm@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "arm" }, "sha512-BWLQO3nemLDSV5PoE5GPHe1dU9Dth77Kv8/cle9Ujcp4LhPo0KincdPqFH/qKeU/xvW25mgFueflZ1nc4rKuww=="], - - "@typescript/native-preview-linux-arm64": ["@typescript/native-preview-linux-arm64@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-VLMEuml3BhUb+jaL0TXQ4xvVODxJF+RhkI+tBWvlynsJI4khTXEiwWh+wPOJrsfBRYFRMXEu28Odl/HXkYze8w=="], - - "@typescript/native-preview-linux-x64": ["@typescript/native-preview-linux-x64@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "x64" }, "sha512-qUrJWTB5/wv4wnRG0TRXElAxc2kykNiRNyEIEqBbLmzDlrcvAW7RRy8MXoY1ZyTiKGMu14itZ3x9oW6+blFpRw=="], - - "@typescript/native-preview-win32-arm64": ["@typescript/native-preview-win32-arm64@7.0.0-dev.20260421.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Rc6NsWlZmCs5YUKVzKgwoBOoRUGsPzct4BDMRX0csD1devLBBc4AbUXWKsJRbpwIAnqMO1ld4sNHEb+wXgfNHQ=="], - - "@typescript/native-preview-win32-x64": ["@typescript/native-preview-win32-x64@7.0.0-dev.20260421.2", "", { "os": "win32", "cpu": "x64" }, "sha512-GQv1+dya1t6EqF2Cpsb+xoozovdX10JUSf6Kl/8xNkTapzmlHd+uMr+8ku3jIASTxoRGn0Mklgjj3MDKrOTuLg=="], - "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], @@ -498,6 +482,8 @@ "twemoji-parser": ["twemoji-parser@14.0.0", "", {}, "sha512-9DUOTGLOWs0pFWnh1p6NF+C3CkQ96PWmEFwhOVmT3WbecRC+68AIqpsnJXygfkFcp4aXbOp8Dwbhh/HQgvoRxA=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + "typescript-telegram-bot-api": ["typescript-telegram-bot-api@0.16.0", "", { "dependencies": { "axios": "^1.7.7", "form-data": "^4.0.1" } }, "sha512-NaAjXucQiZ87U8La/IMaWDOghbMlJzfMbU4rG8ppFpgPOvEwat/zfN5BM+J2QDvKVGN87qJ+1nELnkm3ctSLnQ=="], "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], diff --git a/locales/en.json b/locales/en.json index 8ee942d..c4504fd 100644 --- a/locales/en.json +++ b/locales/en.json @@ -128,6 +128,7 @@ "getImageGenDoneText.default": "👨‍🎨 Image generated.", "getErrorText.withReason": "{errorText} Reason:\n{reason}", "getUseToolText.python": "👨‍💻 Running `Python`", + "getUseToolText.codeInterpreter": "👨‍💻 Running `Code Interpreter`", "getUseToolText.default": "🔧 Using tool `{name}`", "getAnalyzingDocumentText.default": "🔍 Analyzing the document...", "getAnalyzingDocumentText.single": "🔍 Analyzing document: `{name}`", diff --git a/locales/ru.json b/locales/ru.json index 7d22aff..775bb0a 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -8,7 +8,7 @@ }, "providerChoice.default": "По умолчанию", "errorText": "⚠️ Произошла ошибка.", - "waitThinkText": "⏳ Думаю...", + "waitThinkText": "⏳ Дайте-ка подумать...", "analyzingPictureText": "🔍 Анализирую изображение...", "analyzingPicturesText": "🔍 Анализирую изображения...", "reasoningText": "🤔 Рассуждаю...", @@ -154,6 +154,7 @@ "getImageGenDoneText.default": "👨‍🎨 Изображение сгенерировано.", "getErrorText.withReason": "{errorText} Причина:\n{reason}", "getUseToolText.python": "👨‍💻 Запускаю `Python`", + "getUseToolText.codeInterpreter": "👨‍💻 Запускаю `Code Interpreter`", "getUseToolText.default": "🔧 Использую инструмент `{name}`", "getAnalyzingDocumentText.default": "🔍 Анализирую документ...", "getAnalyzingDocumentText.single": "🔍 Анализирую документ: `{name}`", diff --git a/locales/ua.json b/locales/ua.json index 3ca92b6..a92b647 100644 --- a/locales/ua.json +++ b/locales/ua.json @@ -8,7 +8,7 @@ }, "providerChoice.default": "За замовчуванням", "errorText": "⚠️ Сталася помилка.", - "waitThinkText": "⏳ Думаю...", + "waitThinkText": "⏳ Дайте-но подумати...", "analyzingPictureText": "🔍 Аналізую зображення...", "analyzingPicturesText": "🔍 Аналізую зображення...", "reasoningText": "🤔 Міркую...", @@ -153,6 +153,7 @@ "getImageGenDoneText.default": "👨‍🎨 Зображення згенеровано.", "getErrorText.withReason": "{errorText} Причина:\n{reason}", "getUseToolText.python": "👨‍💻 Запускаю `Python`", + "getUseToolText.codeInterpreter": "👨‍💻 Запускаю `Code Interpreter`", "getUseToolText.default": "🔧 Використовую інструмент `{name}`", "getAnalyzingDocumentText.default": "🔍 Аналізую документ...", "getAnalyzingDocumentText.single": "🔍 Аналізую документ: `{name}`", diff --git a/package-lock.json b/package-lock.json index c1a24c1..e71070c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,8 +31,8 @@ "@types/fluent-ffmpeg": "^2.1.28", "@types/node": "^25.6.1", "@types/qrcode": "^1.5.6", - "@typescript/native-preview": "^7.0.0-beta", - "drizzle-kit": "^0.31.10" + "drizzle-kit": "^0.31.10", + "typescript": "^6.0.3" } }, "node_modules/@drizzle-team/brocli": { @@ -2025,123 +2025,6 @@ "@types/node": "*" } }, - "node_modules/@typescript/native-preview": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview/-/native-preview-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-CmajHI25HpVWE9R1XFoxr+cphJPxoYD3eFioQtAvXYkMFKnLdICMS9pXre9Pybizb75ejRxjKD5/CVG055rEIg==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsgo": "bin/tsgo.js" - }, - "optionalDependencies": { - "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260421.2", - "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260421.2", - "@typescript/native-preview-linux-arm": "7.0.0-dev.20260421.2", - "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260421.2", - "@typescript/native-preview-linux-x64": "7.0.0-dev.20260421.2", - "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260421.2", - "@typescript/native-preview-win32-x64": "7.0.0-dev.20260421.2" - } - }, - "node_modules/@typescript/native-preview-darwin-arm64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-darwin-arm64/-/native-preview-darwin-arm64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-fHv1r3ZmVo6zxuAIFmuX3w9QxbcauoG0SsWhmDwm6VmRubLlOJIcmTtlmV3JAb9oOnq8LuzZljzT7Q39fSMQDw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@typescript/native-preview-darwin-x64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-darwin-x64/-/native-preview-darwin-x64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-KWTR6xbW9t+JS7D5DQIzo75pqVXVWUxF9PMv/+S6xsnOjCVd6g0ixHcFpFMJMKSUQpGPr8Z5f7b8ks6LHW01jg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@typescript/native-preview-linux-arm": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-linux-arm/-/native-preview-linux-arm-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-BWLQO3nemLDSV5PoE5GPHe1dU9Dth77Kv8/cle9Ujcp4LhPo0KincdPqFH/qKeU/xvW25mgFueflZ1nc4rKuww==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@typescript/native-preview-linux-arm64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-linux-arm64/-/native-preview-linux-arm64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-VLMEuml3BhUb+jaL0TXQ4xvVODxJF+RhkI+tBWvlynsJI4khTXEiwWh+wPOJrsfBRYFRMXEu28Odl/HXkYze8w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@typescript/native-preview-linux-x64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-linux-x64/-/native-preview-linux-x64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-qUrJWTB5/wv4wnRG0TRXElAxc2kykNiRNyEIEqBbLmzDlrcvAW7RRy8MXoY1ZyTiKGMu14itZ3x9oW6+blFpRw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@typescript/native-preview-win32-arm64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-win32-arm64/-/native-preview-win32-arm64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-Rc6NsWlZmCs5YUKVzKgwoBOoRUGsPzct4BDMRX0csD1devLBBc4AbUXWKsJRbpwIAnqMO1ld4sNHEb+wXgfNHQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@typescript/native-preview-win32-x64": { - "version": "7.0.0-dev.20260421.2", - "resolved": "https://registry.npmjs.org/@typescript/native-preview-win32-x64/-/native-preview-win32-x64-7.0.0-dev.20260421.2.tgz", - "integrity": "sha512-GQv1+dya1t6EqF2Cpsb+xoozovdX10JUSf6Kl/8xNkTapzmlHd+uMr+8ku3jIASTxoRGn0Mklgjj3MDKrOTuLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -4007,6 +3890,20 @@ "integrity": "sha512-9DUOTGLOWs0pFWnh1p6NF+C3CkQ96PWmEFwhOVmT3WbecRC+68AIqpsnJXygfkFcp4aXbOp8Dwbhh/HQgvoRxA==", "license": "MIT" }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/typescript-telegram-bot-api": { "version": "0.16.0", "resolved": "https://registry.npmjs.org/typescript-telegram-bot-api/-/typescript-telegram-bot-api-0.16.0.tgz", diff --git a/package.json b/package.json index ad80620..0ec9ac6 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,6 @@ "@types/node": "^25.6.1", "@types/qrcode": "^1.5.6", "drizzle-kit": "^0.31.10", - "@typescript/native-preview": "^7.0.0-beta" + "typescript": "^6.0.3" } } diff --git a/src/ai/ai-runtime-target.ts b/src/ai/ai-runtime-target.ts index 4819675..623412b 100644 --- a/src/ai/ai-runtime-target.ts +++ b/src/ai/ai-runtime-target.ts @@ -28,6 +28,7 @@ const PURPOSE_SUFFIXES: Record = { thinking: ["THINKING", "THINK"], extendedThinking: ["EXTENDED_THINKING", "THINKING", "THINK"], tools: ["TOOLS", "CHAT"], + toolRank: ["TOOL_RANK", "TOOL_RANKER"], audio: ["AUDIO"], documents: ["DOCUMENTS", "RAG", "EMBEDDING"], outputImages: ["OUTPUT_IMAGES", "IMAGE"], diff --git a/src/ai/provider-model-runtime.ts b/src/ai/provider-model-runtime.ts index 6f0b560..fd65524 100644 --- a/src/ai/provider-model-runtime.ts +++ b/src/ai/provider-model-runtime.ts @@ -111,6 +111,12 @@ function isOpenAiReasoningModel(model: string): boolean { return /^o\d/.test(name) || name.startsWith("gpt-5"); } +// TODO: OpenAI image input rollout +// 1. Keep OpenAI chat payload building `input_image` parts from `MessagePart.imageParts`. +// 2. Replace name-only vision detection with a dedicated OpenAI vision-capability helper. +// 3. Use allowlist/denylist heuristics for `vision` and `ocr`, with a probe/cache fallback later if needed. +// 4. Gate `rejectUnsupportedAttachments()` on the resolved vision capability, not on `OPENAI_IMAGE_MODEL`. +// 5. Add tests for supported/unsupported model names and the resulting `input_image` payload shape. function isOpenAiVisionModel(model: string): boolean { const name = lowerModelName(model); if (!isOpenAiTextModel(model)) return false; diff --git a/src/ai/tool-mappers.ts b/src/ai/tool-mappers.ts index 8d5f39d..2c58c6c 100644 --- a/src/ai/tool-mappers.ts +++ b/src/ai/tool-mappers.ts @@ -1,6 +1,8 @@ import {AiTool} from "./tool-types"; import {AiProvider} from "../model/ai-provider"; import {getTools} from "./tools/registry"; +import {WEB_SEARCH_TOOL_NAME} from "./tools/brave-search"; +import {PYTHON_INTERPRETER_TOOL_NAME} from "./tools/python-interpretator"; export type AiProviderName = "ollama" | "openai" | "gemini" | "mistral"; @@ -8,8 +10,17 @@ export function getOllamaTools(): AiTool[] { return getTools(); } +const openAiForbiddenTools = [ + WEB_SEARCH_TOOL_NAME, + PYTHON_INTERPRETER_TOOL_NAME +] + +function allowedOpenAiTool(tool: AiTool): boolean { + return !openAiForbiddenTools.includes(tool.function.name) +} + export function getOpenAITools(): AiTool[] { - return getTools().map(tool => ({ + return getTools().filter(allowedOpenAiTool).map(tool => ({ type: "function", function: tool.function, })); @@ -23,8 +34,17 @@ export type OpenAiResponseTool = { strict: false; }; +export type OpenAiCodeInterpreterTool = { + type: "code_interpreter"; + container: { + type: "auto"; + file_ids?: string[]; + memory_limit?: "1g" | "4g" | "16g" | "64g" | null; + } | string; +}; + export function getOpenAIResponsesTools(): OpenAiResponseTool[] { - return getTools().map(tool => ({ + return getTools().filter(allowedOpenAiTool).map(tool => ({ type: "function", name: tool.function.name, description: tool.function.description, @@ -33,6 +53,15 @@ export function getOpenAIResponsesTools(): OpenAiResponseTool[] { })); } +export function getOpenAICodeInterpreterTool(): OpenAiCodeInterpreterTool { + return { + type: "code_interpreter", + container: { + type: "auto", + }, + }; +} + export function getMistralTools(): AiTool[] { return getTools().map(tool => ({ type: "function", diff --git a/src/ai/tools/brave-search.ts b/src/ai/tools/brave-search.ts index 74f7468..81a48aa 100644 --- a/src/ai/tools/brave-search.ts +++ b/src/ai/tools/brave-search.ts @@ -90,10 +90,12 @@ type BraveSearchApiResponse = { summarizer?: unknown; }; +export const WEB_SEARCH_TOOL_NAME = "web_search"; + export const braveSearchTool = { type: "function", function: { - name: "web_search", + name: WEB_SEARCH_TOOL_NAME, description: "Search the web using Brave Search API. Use this for current information, facts, documentation, news, products, recent events, source lookup, and general web search. Returns ranked web/news/video results with titles, URLs and snippets.", parameters: { diff --git a/src/ai/tools/list-notes.ts b/src/ai/tools/list-notes.ts index 138bd7c..8558436 100644 --- a/src/ai/tools/list-notes.ts +++ b/src/ai/tools/list-notes.ts @@ -70,7 +70,7 @@ export async function listNotes(): Promise { const markdownFiles = entries .filter((entry) => entry.isFile()) .map((entry) => entry.name) - .filter((fileName) => fileName.endsWith(".md")); + .filter((fileName) => fileName.endsWith(".md") && !fileName.startsWith("index")); const notes: NoteListItem[] = await Promise.all( markdownFiles.map(async (fileName) => { @@ -115,6 +115,10 @@ export async function getNoteContent( return {success: false, error: "No file name provided"}; } + if (fileName.trim().includes("index")) { + return {success: false, error: "It is forbidden to access `index.md`"}; + } + const noteFilePath = buildSafeNoteFilePath(fileName); if (!noteFilePath) { return {success: false, error: "Invalid or unsafe file name provided"}; @@ -125,7 +129,12 @@ export async function getNoteContent( const normalizedFileName = path.basename(noteFilePath); const relativePath = path.relative(path.dirname(notesRootFile), noteFilePath); - logger.debug("get_content.done", {fileName: normalizedFileName, relativePath, chars: content.length, duration: logger.duration(startedAt)}); + logger.debug("get_content.done", { + fileName: normalizedFileName, + relativePath, + chars: content.length, + duration: logger.duration(startedAt) + }); return { success: true, fileName: normalizedFileName, @@ -210,14 +219,14 @@ export const deleteNoteTool = { type: "function", function: { name: "delete_note", - description: "Delete an existing Markdown note by its file name and remove its link from the notes root file if present.", + description: "Delete an existing Markdown note by its file name and remove its link from the notes root file if present. It is forbidden to delete/edit/rename `index.md` note.", parameters: { type: "object", properties: { fileName: { type: "string", description: - "The file name of the note to delete. It may be provided with or without the .md extension. Must not contain forbidden or unsafe characters such as /, \\, :, *, ?, \", <, >, |, or control characters.", + "The file name of the note to delete. It may be provided with or without the .md extension. Must not contain forbidden or unsafe characters such as /, \\, :, *, ?, \", <, >, |, or control characters. It is forbidden to delete/edit/rename `index.md` note.", }, }, required: ["fileName"], @@ -236,6 +245,10 @@ export async function updateNoteContent( return {success: false, error: "No file name provided"}; } + if (fileName.trim().includes("index")) { + return {success: false, error: "It is forbidden to edit `index.md`"}; + } + const content = asNonEmptyString(args?.content) ?? ""; if (!content.trim().length) { return {success: false, error: "No content provided"}; @@ -249,7 +262,12 @@ export async function updateNoteContent( try { await readFile(noteFilePath, "utf-8"); await writeFile(noteFilePath, content, "utf-8"); - logger.debug("update_content.done", {fileName, filePath: noteFilePath, chars: content.length, duration: logger.duration(startedAt)}); + logger.debug("update_content.done", { + fileName, + filePath: noteFilePath, + chars: content.length, + duration: logger.duration(startedAt) + }); return {success: true, filePath: noteFilePath}; } catch (error) { @@ -270,6 +288,10 @@ export async function deleteNote( return {success: false, error: "No file name provided"}; } + if (fileName.trim().includes("index")) { + return {success: false, error: "It is forbidden to delete `index.md`"}; + } + const noteFilePath = buildSafeNoteFilePath(fileName); if (!noteFilePath) { return {success: false, error: "Invalid or unsafe file name provided"}; diff --git a/src/ai/tools/market-rates.ts b/src/ai/tools/market-rates.ts index e5bc317..fe0e763 100644 --- a/src/ai/tools/market-rates.ts +++ b/src/ai/tools/market-rates.ts @@ -4,10 +4,12 @@ import {toolsLogger} from "./tool-logger"; const logger = toolsLogger.child("market-rates"); -export const getMarketRatesTool = { +export const GET_FINANCIAL_MARKET_DATA = "get_financial_market_data"; + +export const getFinancialMarketData = { type: "function", function: { - name: "get_market_rates", + name: GET_FINANCIAL_MARKET_DATA, description: "Retrieve the latest exchange rates for supported currency, crypto, and precious metal pairs, including 24-hour change data when available. Supported pairs: USD/RUB, USD/EUR, USD/KZT, USD/UAH, USD/BYN, USD/GBP, USD/CNY, TON/USD, BTC/USD, ETH/USD, SOL/USD, and XAU/USD. Use this tool when the user asks for current rates, currency conversion, crypto prices, gold price, or recent 24-hour movement. This tool takes no parameters.", parameters: { @@ -18,11 +20,11 @@ export const getMarketRatesTool = { }, } satisfies AiTool; -export const marketRatesToolPrompt = [ +export const financialMarketDataToolPrompt = [ "Currency rates tool rules:", - "- Use `get_market_rates` whenever the answer depends on current exchange rates, crypto prices, or gold price.", - "- Use `get_market_rates` when the user asks whether a supported asset went up or down recently.", - "- Use `get_market_rates` when the user asks for the 24-hour change, percentage change, or movement direction for a supported pair.", + `- Use \`${GET_FINANCIAL_MARKET_DATA}\` whenever the answer depends on current exchange rates, crypto prices, or gold price.`, + `- Use \`${GET_FINANCIAL_MARKET_DATA}\` when the user asks whether a supported asset went up or down recently.`, + `- Use \`${GET_FINANCIAL_MARKET_DATA}\` when the user asks for the 24-hour change, percentage change, or movement direction for a supported pair.`, "- Never guess current rates, prices, or 24-hour changes. Call the tool first.", "- Do not use this tool for unsupported pairs unless the user asks about one of the supported pairs listed below.", "- Do not use this tool for historical rates beyond the provided 24-hour comparison.", diff --git a/src/ai/tools/python-interpretator.ts b/src/ai/tools/python-interpretator.ts index f724dbb..7302dc8 100644 --- a/src/ai/tools/python-interpretator.ts +++ b/src/ai/tools/python-interpretator.ts @@ -4,7 +4,6 @@ import os from "node:os"; import path from "node:path"; import {AiTool} from "../tool-types"; import {Environment} from "../../common/environment"; -import {randomUUID} from "node:crypto"; import {toolsLogger} from "./tool-logger"; const logger = toolsLogger.child("python-interpreter"); @@ -306,7 +305,8 @@ async function executePythonCode( const maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS; // const tempDir = path.join(Environment.DATA_PATH, "cache", "python", "python-temp-" + randomUUID()); - const tempDir = path.join(Environment.FILE_TOOLS_ROOT_DIR ?? ".", "ollama-python-temp-" + randomUUID()); + // const tempDir = path.join(Environment.FILE_TOOLS_ROOT_DIR ?? ".", "ollama-python-temp-" + randomUUID()); + const tempDir = path.join(Environment.FILE_TOOLS_ROOT_DIR ?? ".", ""); const inputDir = path.join(tempDir, PYTHON_INPUTS_DIR_NAME); const outputDir = path.join(tempDir, PYTHON_OUTPUTS_DIR_NAME); const attachmentsPath = path.join(tempDir, PYTHON_ATTACHMENTS_FILE_NAME); @@ -350,7 +350,12 @@ async function executePythonCode( }, }); - logger.debug("process.done", {duration: logger.duration(startedAt), exitCode: result.exitCode, timedOut: result.timedOut, outputTruncated: result.outputTruncated}); + logger.debug("process.done", { + duration: logger.duration(startedAt), + exitCode: result.exitCode, + timedOut: result.timedOut, + outputTruncated: result.outputTruncated + }); if (result.timedOut) { logger.warn("process.timeout", {duration: logger.duration(startedAt)}); @@ -369,7 +374,11 @@ async function executePythonCode( } if (result.outputTruncated) { - logger.warn("process.output_truncated", {duration: logger.duration(startedAt), stdoutChars: result.stdout.length, stderrChars: result.stderr.length}); + logger.warn("process.output_truncated", { + duration: logger.duration(startedAt), + stdoutChars: result.stdout.length, + stderrChars: result.stderr.length + }); return { ok: false, diff --git a/src/ai/tools/registry.ts b/src/ai/tools/registry.ts index cc656d9..b8261d5 100644 --- a/src/ai/tools/registry.ts +++ b/src/ai/tools/registry.ts @@ -5,7 +5,12 @@ import {getCurrentDateTime, getCurrentDateTimeTool} from "./datetime"; import {shellExecute, shellExecuteTool} from "./shell"; import {ToolHandler} from "./types"; import {getWeather, getWeatherTool} from "./weather"; -import {getMarketRates, getMarketRatesTool} from "./market-rates"; +import { + financialMarketDataToolPrompt, + GET_FINANCIAL_MARKET_DATA, + getFinancialMarketData, + getMarketRates +} from "./market-rates"; import {pythonInterpreterTool, runPythonInterpreter} from "./python-interpretator"; import { copyPath, @@ -36,19 +41,19 @@ import { updateNoteContent, updateNoteContentTool } from "./list-notes"; -import {getNoteFile, getNoteFileTool} from "./send-note-file"; +import {sendNoteAsFileTool, sendNoteAsFile} from "./send-note-as-file"; import {searchNotes, searchNotesTool} from "./search-notes"; export const getTools = () => { const tools: AiTool[] = [ getCurrentDateTimeTool, - getMarketRatesTool, + getFinancialMarketData, createNoteTool, listNotesTool, getNoteContentTool, updateNoteContentTool, deleteNoteTool, - getNoteFileTool, + sendNoteAsFileTool, searchNotesTool ]; @@ -97,13 +102,13 @@ export const getTools = () => { export const getToolHandlers = () => { let handlers: Record = { get_datetime: getCurrentDateTime, - get_market_rates: getMarketRates, + get_financial_market_data: getMarketRates, create_note: createNote, list_notes: listNotes, get_note_content: getNoteContent, update_note_content: updateNoteContent, delete_note: deleteNote, - get_note_file: getNoteFile, + send_note_as_file: sendNoteAsFile, search_notes: searchNotes }; @@ -151,3 +156,19 @@ export const getToolHandlers = () => { return handlers; }; + +export function getToolPrompts(toolNames: string[]): string[] { + const prompts: string[] = []; + + for (const toolName of toolNames) { + switch (toolName) { + case GET_FINANCIAL_MARKET_DATA: + prompts.push(financialMarketDataToolPrompt); + break; + default: + break; + } + } + + return prompts; +} \ No newline at end of file diff --git a/src/ai/tools/send-note-file.ts b/src/ai/tools/send-note-as-file.ts similarity index 97% rename from src/ai/tools/send-note-file.ts rename to src/ai/tools/send-note-as-file.ts index 678c1d8..6dddac2 100644 --- a/src/ai/tools/send-note-file.ts +++ b/src/ai/tools/send-note-as-file.ts @@ -44,10 +44,10 @@ export const GetNoteFileResultSchema = z.discriminatedUnion("success", [ }), ]); -export const getNoteFileTool = { +export const sendNoteAsFileTool = { type: "function", function: { - name: "get_note_file", + name: "send_note_as_file", description: "Prepare a Markdown note file to be sent to the user as a .md attachment. Returns a local file descriptor that the host application should use to upload or send the file.", parameters: { @@ -64,7 +64,7 @@ export const getNoteFileTool = { }, } satisfies AiTool; -export async function getNoteFile( +export async function sendNoteAsFile( args?: Record, ): Promise { logger.debug("start", {args}); diff --git a/src/ai/tools/shell.ts b/src/ai/tools/shell.ts index f8331b0..97e113f 100644 --- a/src/ai/tools/shell.ts +++ b/src/ai/tools/shell.ts @@ -6,7 +6,7 @@ export const shellExecuteTool = { type: "function", function: { name: "shell_execute", - description: "Execute command in a shell", + description: "Execute NON-Python command in a shell. Do not use if you intend to execute some python.", parameters: { type: "object", properties: { diff --git a/src/ai/tools/utils.ts b/src/ai/tools/utils.ts index 6dfba61..b10cea8 100644 --- a/src/ai/tools/utils.ts +++ b/src/ai/tools/utils.ts @@ -1,5 +1,4 @@ import {Ollama} from "ollama"; -import {z} from "zod"; import {toolsLogger} from "./tool-logger"; const logger = toolsLogger.child("utils"); @@ -105,26 +104,4 @@ export async function loadOllamaModel(model: string, ollama: Ollama, contextLeng logger.error("ollama.load.failed", {model, contextLength, error: e}); return false; } -} - -export type ToolPlanStep = { - t: string; - h: string; - from: string; -}; - -export type RouterPlan = { - s: ToolPlanStep[]; - m: string; -}; - -export const ToolPlanStepSchema = z.object({ - t: z.string(), - h: z.string(), - from: z.string(), -}); - -export const RouterPlanSchema = z.object({ - s: z.array(ToolPlanStepSchema), - m: z.string() -}); \ No newline at end of file +} \ No newline at end of file diff --git a/src/ai/unified-ai-runner.ollama.ts b/src/ai/unified-ai-runner.ollama.ts index c3feba2..6cc04e8 100644 --- a/src/ai/unified-ai-runner.ollama.ts +++ b/src/ai/unified-ai-runner.ollama.ts @@ -2,27 +2,48 @@ import {Message} from "typescript-telegram-bot-api"; import * as fs from "node:fs"; import path from "node:path"; -import {AiProvider} from "../model/ai-provider"; import {Environment} from "../common/environment"; import {bot, notesDir} from "../index"; import {clamp, logError} from "../util/utils"; import {getOllamaTools} from "./tool-mappers"; import {TelegramStreamMessage} from "./telegram-stream-message"; -import {getModelCapabilities} from "./provider-model-runtime"; import {ChatMessage} from "./chat-messages-types"; import {ChatRequest, Tool} from "ollama"; import {ToolRuntimeContext} from "./tools/runtime"; import {enqueueTelegramApiCall} from "../util/telegram-api-queue"; import {getCurrentDateTimeTool} from "./tools/datetime"; -import {getMarketRatesTool} from "./tools/market-rates"; +import {getFinancialMarketData} from "./tools/market-rates"; import {getWeatherTool} from "./tools/weather"; import {loadOllamaModel, unloadAllOllamaModels} from "./tools/utils"; import {createOllamaClient} from "./ai-runtime-target"; -import {GetNoteFileResult, GetNoteFileResultSchema} from "./tools/send-note-file"; +import {GetNoteFileResult, GetNoteFileResultSchema, sendNoteAsFileTool} from "./tools/send-note-as-file"; import {aiLog, aiLogDuration, aiLogMessageIdentity, aiLogProviderTarget, aiLogToolCall} from "../logging/ai-logger"; -import {DEFAULT_OLLAMA_CONTEXT_SIZE, MAX_OLLAMA_CONTEXT_SIZE, MAX_TOOL_ROUNDS, MIN_OLLAMA_CONTEXT_SIZE, RuntimeConfigSnapshot, Think, ToolCallData, ToolExecutionMemory, allToolSchemaNames, appendOllamaToolResults, dedupeToolCalls, executeToolBatch, normalizeOllamaToolCalls, roundStatus, safeJsonParseObject, isRecord, isOllamaModelActive, OllamaToolCallLike} from "./unified-ai-runner.shared"; -import {latestUserTextFromOllamaMessages, looksLikeToolRankerJson, OllamaToolRanker} from "./unified-ai-runner.tool-ranker"; +import { + allToolSchemaNames, + appendOllamaToolResults, + dedupeToolCalls, + DEFAULT_OLLAMA_CONTEXT_SIZE, + executeToolBatch, + isOllamaModelActive, + isRecord, + MAX_OLLAMA_CONTEXT_SIZE, + MAX_TOOL_ROUNDS, + MIN_OLLAMA_CONTEXT_SIZE, + normalizeOllamaToolCalls, + OllamaToolCallLike, + roundStatus, + RuntimeConfigSnapshot, + safeJsonParseObject, + Think, + ToolCallData, + ToolExecutionMemory +} from "./unified-ai-runner.shared"; +import {latestUserTextFromOllamaMessages, OllamaToolRanker} from "./unified-ai-runner.tool-ranker"; +import {getToolPrompts} from "./tools/registry"; +import {createNoteTool} from "./tools/create-note"; +import {deleteNoteTool, getNoteContentTool, listNotesTool, updateNoteContentTool} from "./tools/list-notes"; +import {searchNotesTool} from "./tools/search-notes"; export async function runOllama( msg: Message, @@ -156,46 +177,76 @@ export async function runOllama( messages: messages, think: audioCount ? false : think, options: { - temperature: 0.6, - num_ctx: context, + temperature: 0.7, + top_p: 0.9, + top_k: 40, + num_ctx: 16384 } }; let activeToolNames: string[] = []; - if ((await getModelCapabilities(AiProvider.OLLAMA, model, "tools"))?.tools?.supported) { - const availableOllamaTools: Tool[] = fromId !== Environment.CREATOR_ID - ? [getCurrentDateTimeTool, getMarketRatesTool, getWeatherTool] - : getOllamaTools() as Tool[]; + // if ((await getModelCapabilities(AiProvider.OLLAMA, model, "tools"))?.tools?.supported) { + const availableOllamaTools: Tool[] = + fromId !== Environment.CREATOR_ID ? [ + getCurrentDateTimeTool, + getFinancialMarketData, + getWeatherTool, + createNoteTool, + listNotesTool, + getNoteContentTool, + updateNoteContentTool, + deleteNoteTool, + sendNoteAsFileTool, + searchNotesTool + ] : getOllamaTools() as Tool[]; - aiLog("debug", "ollama.tools.available", { - round, - tools: allToolSchemaNames(availableOllamaTools), - rankerEnabled: !!config.ollamaToolRankerTarget, - }); + aiLog("debug", "ollama.tools.available", { + round, + tools: allToolSchemaNames(availableOllamaTools), + rankerEnabled: !!config.ollamaToolRankerTarget, + }); - const rankerSelection = await new OllamaToolRanker(config).selectTools({ - userQuery: latestUserTextFromOllamaMessages(messages), - availableTools: availableOllamaTools, - round, - signal, - }); + const rankerSelection = await new OllamaToolRanker(config).selectTools({ + userQuery: latestUserTextFromOllamaMessages(messages), + availableTools: availableOllamaTools, + round, + signal, + }); - activeToolNames = rankerSelection.selectedNames; - if (rankerSelection.tools.length > 0) { - request.tools = rankerSelection.tools; - } else { - delete request.tools; + activeToolNames = rankerSelection.tools.map(t => t.function.name ?? ""); + if (rankerSelection.tools.length > 0) { + request.tools = [...rankerSelection.tools, ...rankerSelection.tools]; + request.options = { + ...request.options, + temperature: 0 } - aiLog("debug", "ollama.tools.selected", { - round, - tools: activeToolNames, - count: activeToolNames.length, - usedRanker: rankerSelection.usedRanker, - missing: rankerSelection.missing, - }); + const newMessage = messages[messages.length - 1]; + if (newMessage) { + newMessage.content += "\n" + "Suggested tools to call: " + activeToolNames.join(", "); + } + + const systemMessage = messages.find(m => m.role === "system"); + if (systemMessage) { + systemMessage.content += "\n\n" + getToolPrompts(activeToolNames).join("\n\n"); + } + + request.model = config.ollamaToolTarget.model; + } else { + delete request.tools; } + // TODO: 14.05.2026, Danil Nikolaev: check if model supports tools + + + aiLog("debug", "ollama.tools.selected", { + round, + tools: activeToolNames, + count: activeToolNames.length, + usedRanker: rankerSelection.usedRanker, + }); + // } + if (!stream) { const response = await ollama.chat({ ...request, @@ -214,14 +265,14 @@ export async function runOllama( const responseText = rawContent; - if (looksLikeToolRankerJson(responseText)) { - aiLog("error", "ollama.response.looks_like_tool_ranker_json", { - round, - preview: responseText.slice(0, 800), - target: aiLogProviderTarget(target), - }); - throw new Error("Ollama chat model returned tool-ranker JSON. Check that OLLAMA chat target and OLLAMA tools/ranker target are not mixed up."); - } + // if (looksLikeToolRankerJson(responseText)) { + // aiLog("error", "ollama.response.looks_like_tool_ranker_json", { + // round, + // preview: responseText.slice(0, 800), + // target: aiLogProviderTarget(target), + // }); + // throw new Error("Ollama chat model returned tool-ranker JSON. Check that OLLAMA chat target and OLLAMA tools/ranker target are not mixed up."); + // } streamMessage.append(responseText); @@ -264,6 +315,7 @@ export async function runOllama( continue; } + console.log("MESSAGES", JSON.stringify(request.messages)); const response = await ollama.chat({ ...request, stream: true @@ -277,6 +329,8 @@ export async function runOllama( if (signal.aborted) abortOllamaResponse(); try { for await (const chunk of response) { + console.log("OLLAMA_CHUNK: ", chunk); + const localToolCalls: ToolCallData[] = []; localToolCalls.push(...normalizeOllamaToolCalls( @@ -324,16 +378,16 @@ export async function runOllama( signal.removeEventListener("abort", abortOllamaResponse); } - const streamedRoundText = streamMessage.getText().slice(roundTextStart); - if (!calls.length && looksLikeToolRankerJson(streamedRoundText)) { - streamMessage.replaceText(streamMessage.getText().slice(0, roundTextStart)); - aiLog("error", "ollama.response.looks_like_tool_ranker_json", { - round, - preview: streamedRoundText.slice(0, 800), - target: aiLogProviderTarget(target), - }); - throw new Error("Ollama chat model returned tool-ranker JSON. Check that OLLAMA chat target and OLLAMA tools/ranker target are not mixed up."); - } + // const streamedRoundText = streamMessage.getText().slice(roundTextStart); + // if (!calls.length && looksLikeToolRankerJson(streamedRoundText)) { + // streamMessage.replaceText(streamMessage.getText().slice(0, roundTextStart)); + // aiLog("error", "ollama.response.looks_like_tool_ranker_json", { + // round, + // preview: streamedRoundText.slice(0, 800), + // target: aiLogProviderTarget(target), + // }); + // throw new Error("Ollama chat model returned tool-ranker JSON. Check that OLLAMA chat target and OLLAMA tools/ranker target are not mixed up."); + // } if (!calls.length) { aiLog("success", "ollama.run.done", { @@ -396,9 +450,4 @@ export async function runOllama( } finally { if (interval) clearInterval(interval); } -} - - -export class OllamaProviderRunner { - static run = runOllama; -} +} \ No newline at end of file diff --git a/src/ai/unified-ai-runner.openai.ts b/src/ai/unified-ai-runner.openai.ts index c4d058f..bb00ff5 100644 --- a/src/ai/unified-ai-runner.openai.ts +++ b/src/ai/unified-ai-runner.openai.ts @@ -21,6 +21,7 @@ import {aiLog, aiLogDuration, aiLogMessageIdentity, aiLogProviderTarget, aiLogTo import { AsyncIterableStream, collectOpenAiResponseFunctionCalls, + collectOpenAiResponseCodeInterpreterCalls, collectOpenAiResponseImages, collectOpenAiResponseText, executeToolBatch, @@ -44,7 +45,7 @@ import { ToolCallData, ToolExecutionMemory } from "./unified-ai-runner.shared"; -import {GetNoteFileResult, GetNoteFileResultSchema} from "./tools/send-note-file"; +import {GetNoteFileResult, GetNoteFileResultSchema} from "./tools/send-note-as-file"; import {bot, notesDir} from "../index"; import fs from "node:fs"; import path from "node:path"; @@ -60,9 +61,11 @@ export async function runOpenAi( sourceMessage: Message, config: RuntimeConfigSnapshot, toolContext: ToolRuntimeContext, + think?: boolean ): Promise { // TODO: 13.05.2026: remove firstRoundStatus; + think; const runnerStartedAt = Date.now(); let responseInput: unknown[] = [...messages]; const openAi = createOpenAiClient(config.openAiChatTarget); @@ -111,6 +114,21 @@ export async function runOpenAi( ); } + const codeInterpreterCalls = collectOpenAiResponseCodeInterpreterCalls(response); + if (codeInterpreterCalls.length) { + aiLog("info", "openai.code_interpreter_calls", { + round, + duration: aiLogDuration(roundStartedAt), + calls: codeInterpreterCalls.map(call => ({ + id: call.id, + status: call.status, + containerId: call.containerId, + codeChars: call.code?.length ?? 0, + outputItems: call.outputs.length, + })), + }); + } + const calls = collectOpenAiResponseFunctionCalls(response); aiLog(calls.length ? "info" : "success", calls.length ? "openai.tool_calls" : "openai.run.done", { round, @@ -170,6 +188,7 @@ export async function runOpenAi( input: responseInput as ResponseInputItem[], stream: true, tools: getOpenAIResponsesToolsWithImage(config) as ResponseCreateParamsStreaming["tools"], + parallel_tool_calls: true }; const response = await openAi.responses.create(request, {signal}) as unknown as AsyncIterableStream; @@ -207,6 +226,18 @@ export async function runOpenAi( streamMessage.setStatus(Environment.finalizingImageGenText); await streamMessage.flush(); break; + case "response.code_interpreter_call.in_progress": + case "response.code_interpreter_call.interpreting": + streamMessage.setStatus(Environment.getUseToolText(["code_interpreter"])); + await streamMessage.flush(); + break; + case "response.code_interpreter_call.completed": + streamMessage.clearStatus(); + await streamMessage.flush(); + break; + case "response.code_interpreter_call_code.delta": + case "response.code_interpreter_call_code.done": + break; case "response.output_item.added": if (event.item.type === "function_call" && event.item.name) { const item = event.item as OpenAiResponseOutputItem & { id?: string }; @@ -275,6 +306,21 @@ export async function runOpenAi( ); } + const codeInterpreterCalls = collectOpenAiResponseCodeInterpreterCalls(completedResponse); + if (codeInterpreterCalls.length) { + aiLog("info", "openai.code_interpreter_calls", { + round, + duration: aiLogDuration(roundStartedAt), + calls: codeInterpreterCalls.map(call => ({ + id: call.id, + status: call.status, + containerId: call.containerId, + codeChars: call.code?.length ?? 0, + outputItems: call.outputs.length, + })), + }); + } + const calls = collectOpenAiResponseFunctionCalls(completedResponse); aiLog(calls.length ? "info" : "success", calls.length ? "openai.tool_calls" : "openai.run.done", { round, @@ -422,7 +468,7 @@ export async function runOpenAiCompatibleChat( model: config.geminiChatTarget.model, messages: chatMessages, tools: getOpenAITools(), - temperature: 0.6, + // temperature: 0.6, }; const response = await geminiOpenAi.chat.completions.create(request, {signal}) as unknown as OpenAiChatCompletionResponseLike; const message = response.choices?.[0]?.message; @@ -484,8 +530,9 @@ export async function runOpenAiCompatibleChat( model: config.geminiChatTarget.model, messages: chatMessages, tools: getOpenAITools(), - temperature: 0.6, + // temperature: 0.6, stream: true, + parallel_tool_calls: true }; const response = await geminiOpenAi.chat.completions.create(request, {signal}) as unknown as AsyncIterableStream; diff --git a/src/ai/unified-ai-runner.shared.ts b/src/ai/unified-ai-runner.shared.ts index 8b870e6..d1e3d61 100644 --- a/src/ai/unified-ai-runner.shared.ts +++ b/src/ai/unified-ai-runner.shared.ts @@ -8,7 +8,7 @@ import {photoGenDir} from "../index"; import {collectReplyChainText, delay, logError, replyToMessage} from "../util/utils"; import {MessageStore} from "../common/message-store"; import type {OpenAiResponseTool} from "./tool-mappers"; -import {AiProviderName, getOpenAIResponsesTools} from "./tool-mappers"; +import {AiProviderName, getOpenAICodeInterpreterTool, getOpenAIResponsesTools} from "./tool-mappers"; import {TelegramArtifactFile, TelegramStreamMessage} from "./telegram-stream-message"; import {AiDownloadedFile} from "./telegram-attachments"; import {getRuntimeCapabilities} from "./provider-model-runtime"; @@ -19,7 +19,8 @@ import {executeToolCall, ToolRuntimeContext} from "./tools/runtime"; import {MessageImagePart, MessagePart} from "../common/message-part"; import {KeyedAsyncLock} from "../util/async-lock"; import {type AiRequestQueueTarget} from "./provider-request-queue"; -import {PYTHON_INTERPRETER_TOOL_NAME, pythonInterpreterToolPrompt} from "./tools/python-interpretator"; +import {PYTHON_INTERPRETER_TOOL_NAME} from "./tools/python-interpretator"; +import {pythonInterpreterToolPrompt} from "./tools/python-interpretator"; import {getResponseLanguageInstruction, UserAiResponseLanguage, UserAiVoiceMode} from "../common/user-ai-settings"; import { isTranscribableAudioDownload, @@ -66,7 +67,7 @@ export type { export type {GenerateContentParameters} from "@google/genai"; export const TELEGRAM_LIMIT = 4096; -export const MAX_TOOL_ROUNDS = 12; +export const MAX_TOOL_ROUNDS = 40; export const MAX_IDENTICAL_TOOL_CALLS = 1; export const OPENAI_IMAGE_PARTIALS = 3; export const AI_REQUEST_TIMEOUT_MS = 10 * 60 * 1000; @@ -163,11 +164,16 @@ export type OpenAiChatToolCallLike = { export type OpenAiResponseOutputItem = { type?: string; + id?: string; call_id?: string; name?: string; arguments?: string; result?: string; content?: Array<{ text?: string; refusal?: string }>; + code?: string | null; + container_id?: string; + outputs?: Array<{ type?: "logs" | "image"; logs?: string; url?: string }> | null; + status?: string; }; export type OpenAiResponseLike = { @@ -265,6 +271,7 @@ export type RuntimeConfigSnapshot = { ollamaChatTarget: AiRuntimeTarget; ollamaToolRankerTarget?: AiRuntimeTarget; + ollamaToolTarget: AiRuntimeTarget; ollamaVisionTarget: AiRuntimeTarget; ollamaThinkingTarget: AiRuntimeTarget; ollamaAudioTarget: AiRuntimeTarget; @@ -296,7 +303,8 @@ export function snapshotRuntimeConfig(): RuntimeConfigSnapshot { rankerToolPrompt: Environment.RANKER_TOOL_PROMPT, ollamaChatTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "chat"), - ollamaToolRankerTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "tools"), + ollamaToolRankerTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "toolRank"), + ollamaToolTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "tools"), ollamaVisionTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "vision"), ollamaThinkingTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "thinking"), ollamaAudioTarget: resolveAiRuntimeTarget(AiProvider.OLLAMA, "audio"), @@ -369,6 +377,7 @@ export function providerTargets(provider: AiProvider, config: RuntimeConfigSnaps return [ config.ollamaChatTarget, config.ollamaToolRankerTarget, + config.ollamaToolTarget, config.ollamaVisionTarget, config.ollamaThinkingTarget, config.ollamaAudioTarget, @@ -402,7 +411,7 @@ export function buildSystemInstruction( includePythonToolPrompt: boolean, ): string { return [ - getResponseLanguageInstruction(responseLanguage), + config.useSystemPrompt ? getResponseLanguageInstruction(responseLanguage) : null, config.systemPrompt && config.useSystemPrompt ? config.systemPrompt : null, includePythonToolPrompt ? pythonInterpreterToolPrompt : null, ].filter(Boolean).join("\n\n"); @@ -1624,6 +1633,7 @@ export function allToolSchemaNames(tools: readonly unknown[]): string[] { export function getOpenAIResponsesToolsWithImage(config: RuntimeConfigSnapshot): Array { return [ ...getOpenAIResponsesTools(), + getOpenAICodeInterpreterTool(), { type: "image_generation", model: config.openAiImageTarget.model, @@ -1632,6 +1642,7 @@ export function getOpenAIResponsesToolsWithImage(config: RuntimeConfigSnapshot): output_format: "png", partial_images: OPENAI_IMAGE_PARTIALS, }, + {type: "web_search"} ]; } @@ -1655,6 +1666,26 @@ export function collectOpenAiResponseFunctionCalls(response: OpenAiResponseLike) })); } +export type OpenAiCodeInterpreterCall = { + id: string; + code: string | null; + containerId: string; + status: string; + outputs: Array<{type?: "logs" | "image"; logs?: string; url?: string}>; +}; + +export function collectOpenAiResponseCodeInterpreterCalls(response: OpenAiResponseLike): OpenAiCodeInterpreterCall[] { + return (response.output ?? []) + .filter(item => item.type === "code_interpreter_call" && item.id && item.container_id) + .map(item => ({ + id: item.id!, + code: item.code ?? null, + containerId: item.container_id!, + status: item.status ?? "unknown", + outputs: Array.isArray(item.outputs) ? item.outputs : [], + })); +} + export function collectOpenAiResponseImages(response: OpenAiResponseLike): string[] { return (response.output ?? []) .filter(item => item.type === "image_generation_call" && typeof item.result === "string") diff --git a/src/ai/unified-ai-runner.tool-ranker.ts b/src/ai/unified-ai-runner.tool-ranker.ts index 482506b..18b2688 100644 --- a/src/ai/unified-ai-runner.tool-ranker.ts +++ b/src/ai/unified-ai-runner.tool-ranker.ts @@ -1,29 +1,23 @@ import {Tool} from "ollama"; -import {AiRuntimeTarget, createOllamaClient} from "./ai-runtime-target"; +import {createOllamaClient} from "./ai-runtime-target"; import {aiLog, aiLogDuration, aiLogProviderTarget} from "../logging/ai-logger"; -import {allToolSchemaNames, isRecord, JsonObject, RuntimeConfigSnapshot, safeJsonParseObject, toolSchemaNames} from "./unified-ai-runner.shared"; - -type RankedToolStep = { - t: string | string[]; - h?: string; - from?: string; -}; - -type RankedToolPlan = { - s?: RankedToolStep[]; - m?: string; -}; +import { + allToolSchemaNames, + isRecord, + RuntimeConfigSnapshot, + toolSchemaNames +} from "./unified-ai-runner.shared"; +import {z} from "zod"; +import {PYTHON_INTERPRETER_TOOL_NAME} from "./tools/python-interpretator"; export type ToolRankerSelection = { tools: Tool[]; - selectedNames: string[]; - missing: string; - raw: string; usedRanker: boolean; }; export class OllamaToolRanker { - constructor(private readonly config: RuntimeConfigSnapshot) {} + constructor(private readonly config: RuntimeConfigSnapshot) { + } async selectTools(args: { userQuery: string; @@ -35,27 +29,71 @@ export class OllamaToolRanker { const target = this.config.ollamaToolRankerTarget; if (!availableTools.length) { - return {tools: [], selectedNames: [], missing: "", raw: "", usedRanker: false}; + return {tools: [], usedRanker: false}; } // Ranker disabled/unconfigured: keep old behavior and expose all allowed tools. if (!target?.model) { return { tools: availableTools, - selectedNames: allToolSchemaNames(availableTools), - missing: "", - raw: "", usedRanker: false, }; } const startedAt = Date.now(); const availableNames = new Set(allToolSchemaNames(availableTools)); - const prompt = this.config.rankerToolPrompt?.trim() || DEFAULT_TOOL_RANKER_PROMPT; - const toolsForPrompt = availableTools.map(tool => ({ - names: toolSchemaNames(tool), - schema: tool, - })); + // const prompt = this.config.rankerToolPrompt?.trim() || DEFAULT_TOOL_RANKER_PROMPT; + + const availableToolNames = availableTools.map(t => "- " + (t.function.name ?? "")); + + const toolRouterPrompt = () => [ + "You are a tool routing model.", + "Select the best tool.", + "Return ONLY valid JSON.", + "", + "Available tools:", + "- no_tool", + // "- ask_clarification", + // "- user_sad", + // "- user_angry", + availableToolNames.join("\n"), + "", + "Never explain reasoning.", + // "If user is sounds aggressive/angry, then pick `user_angry` tool.", + // "If the user's request is unclear, then pick `ask_clarification` tool.", + availableToolNames.find(t => t === "web_search") ? + "If you don't know the answer, then pick `search_web` tool." : null, + availableToolNames.find(t => t == PYTHON_INTERPRETER_TOOL_NAME) ? + "If user asks to write/execute Python code, then use `" + PYTHON_INTERPRETER_TOOL_NAME + "`" : null, + "", + + "Return valid JSON ONLY in this format (NO ARGUMENTS): {\"toolNames\": [\"$toolName\", ... \"lastToolName\"]}", + ].filter(Boolean).join("\n"); + + const routerSchema = { + type: "object", + properties: { + name: { + type: "string", + enum: [ + "no_tool", + "ask_clarification", + ...availableToolNames + ], + }, + arguments: { + type: "object", + additionalProperties: true, + }, + }, + required: ["name"], + additionalProperties: false, + } as const; + + // const toolsForPrompt = availableTools.map(tool => ({ + // names: toolSchemaNames(tool), + // schema: tool, + // })); aiLog("debug", "ollama.tool_ranker.start", { round, @@ -65,46 +103,86 @@ export class OllamaToolRanker { }); try { - const ollama = createOllamaClient(target as AiRuntimeTarget); + const ollama = createOllamaClient(target); + // const response = await ollama.chat({ + // model: target.model, + // messages: [ + // {role: "system", content: prompt}, + // { + // role: "user", + // content: JSON.stringify({ + // q: userQuery, + // tools: toolsForPrompt, + // }), + // }, + // ], + // stream: false, + // options: { + // temperature: 0, + // num_ctx: 8192, + // }, + // }); + + const then = performance.now(); + const response = await ollama.chat({ - model: target.model, + model: target?.model ?? "", messages: [ - {role: "system", content: prompt}, + { + role: "system", + content: toolRouterPrompt() + }, { role: "user", - content: JSON.stringify({ - q: userQuery, - tools: toolsForPrompt, - }), - }, + content: userQuery, + } ], stream: false, + think: false, + format: routerSchema, options: { temperature: 0, + top_p: 0.8, + top_k: 20, + repeat_penalty: 1.05, num_ctx: 8192, + num_predict: 256 }, }); + const now = performance.now(); + const diff = now - then; + console.log("TOOK " + diff + "ms"); + + console.log("OLLAMA_RESPONSE: ", JSON.stringify(response)); + if (signal.aborted) throw new Error("Aborted"); const raw = response.message?.content?.trim() ?? ""; - const plan = parseToolRankerPlan(raw); - const selectedNames = normalizeToolRankerNames(plan, availableNames); + const schema = z.object({ + toolNames: z.array(z.string()) + }); + const res = schema.safeParse(JSON.parse(raw)); + + const selectedNames: string[] = []; + + if (res.success) { + selectedNames.push(...res.data.toolNames); + } + const selectedNameSet = new Set(selectedNames); const tools = availableTools.filter(tool => toolSchemaNames(tool).some(name => selectedNameSet.has(name))); - const missing = typeof plan?.m === "string" ? plan.m.trim() : ""; aiLog("debug", "ollama.tool_ranker.done", { round, duration: aiLogDuration(startedAt), selectedNames, selectedCount: tools.length, - missing, rawPreview: raw.slice(0, 800), }); // Important: if the plan is valid and empty, use NO tools. Do not fall back to all tools. - return {tools, selectedNames, missing, raw, usedRanker: true}; + return {tools, usedRanker: true}; } catch (error) { if (String(error).includes("Aborted")) throw error; @@ -119,9 +197,6 @@ export class OllamaToolRanker { // In that case, preserve availability rather than silently disabling tools. return { tools: availableTools, - selectedNames: allToolSchemaNames(availableTools), - missing: "", - raw: "", usedRanker: false, }; } @@ -141,85 +216,4 @@ export function latestUserTextFromOllamaMessages(messages: readonly { role?: str } } return ""; -} - -export function looksLikeToolRankerJson(text: string): boolean { - const parsed = safeJsonParseObject(extractJsonObjectText(text) ?? text); - return Array.isArray(parsed.s) && typeof parsed.m === "string"; -} - -function parseToolRankerPlan(raw: string): RankedToolPlan | undefined { - const jsonText = extractJsonObjectText(raw); - if (!jsonText) return undefined; - - const parsed = safeJsonParseObject(jsonText) as JsonObject; - if (!Array.isArray(parsed.s)) return undefined; - - return parsed as RankedToolPlan; -} - -function normalizeToolRankerNames(plan: RankedToolPlan | undefined, availableNames: Set): string[] { - if (!plan?.s?.length) return []; - - const result: string[] = []; - for (const step of plan.s) { - const rawNames = Array.isArray(step.t) ? step.t : [step.t]; - for (const rawName of rawNames) { - if (typeof rawName !== "string") continue; - const name = rawName.trim(); - if (availableNames.has(name) && !result.includes(name)) { - result.push(name); - } - } - } - return result; -} - -function extractJsonObjectText(raw: string): string | undefined { - const text = raw.trim() - .replace(/^```(?:json)?\s*/i, "") - .replace(/\s*```$/i, "") - .trim(); - - const start = text.indexOf("{"); - if (start === -1) return undefined; - - let depth = 0; - let inString = false; - let escaped = false; - - for (let i = start; i < text.length; i++) { - const ch = text[i]; - - if (escaped) { - escaped = false; - continue; - } - - if (ch === "\\") { - escaped = true; - continue; - } - - if (ch === '"') { - inString = !inString; - continue; - } - - if (inString) continue; - - if (ch === "{") depth++; - if (ch === "}") depth--; - - if (depth === 0) { - return text.slice(start, i + 1); - } - } - - return undefined; -} - -const DEFAULT_TOOL_RANKER_PROMPT = `You are a tool router. Return strict compact JSON only. -Schema: {"s":[{"t":"tool_name","h":"short input hint","from":"previous_tool.output_or_empty"}],"m":""} -Use tools only when they are needed. If no tool is needed, return {"s":[],"m":""}. -Never answer the user. Never explain. Never use markdown.`; +} \ No newline at end of file diff --git a/src/ai/unified-ai-runner.ts b/src/ai/unified-ai-runner.ts index 10882b2..ee35b2e 100644 --- a/src/ai/unified-ai-runner.ts +++ b/src/ai/unified-ai-runner.ts @@ -8,7 +8,13 @@ import {AiDownloadedFile, attachmentsToDownloadedFiles, cleanupDownloads} from " import {ChatMessage} from "./chat-messages-types"; import {aiProviderRequestQueue} from "./provider-request-queue"; import {prepareOllamaDocumentRag} from "./ollama-rag"; -import {AI_VOICE_MODE_TRANSCRIPT, DEFAULT_AI_RESPONSE_LANGUAGE, resolveAiContextSizeForUser, resolveAiResponseLanguageForUser, resolveAiVoiceModeForUser} from "../common/user-ai-settings"; +import { + AI_VOICE_MODE_TRANSCRIPT, + DEFAULT_AI_RESPONSE_LANGUAGE, + resolveAiContextSizeForUser, + resolveAiResponseLanguageForUser, + resolveAiVoiceModeForUser +} from "../common/user-ai-settings"; import {isTranscribableAudioDownload} from "./speech-to-text"; import {OpenAIChatMessage} from "./openai-chat-message"; import {MistralChatMessage} from "./mistral-chat-message"; @@ -22,7 +28,29 @@ import {runOpenAi, runOpenAiCompatibleChat} from "./unified-ai-runner.openai"; import {runOllama} from "./unified-ai-runner.ollama"; import {runMistral} from "./unified-ai-runner.mistral"; import {runGemini} from "./unified-ai-runner.gemini"; -import {AI_REQUEST_TIMEOUT_MS, TELEGRAM_LIMIT, RuntimeConfigSnapshot, UnifiedRunOptions, appendTranscriptToChatMessages, collectCachedMessageAttachments, collectRequestedAttachmentKinds, collectTextMessages, deleteMistralLibrary, hasAudioAttachmentKind, initialStatus, isAbortError, prepareMistralDocuments, providerName, rejectUnsupportedAttachments, resolveAiRequestQueueTarget, snapshotModel, snapshotRuntimeConfig, stripAudioFromRunnerMessages, toolRuntimeContextFromDownloads, transcribeAudioIfNeeded} from "./unified-ai-runner.shared"; +import { + AI_REQUEST_TIMEOUT_MS, + appendTranscriptToChatMessages, + collectCachedMessageAttachments, + collectRequestedAttachmentKinds, + collectTextMessages, + deleteMistralLibrary, + hasAudioAttachmentKind, + initialStatus, + isAbortError, + prepareMistralDocuments, + providerName, + rejectUnsupportedAttachments, + resolveAiRequestQueueTarget, + RuntimeConfigSnapshot, + snapshotModel, + snapshotRuntimeConfig, + stripAudioFromRunnerMessages, + TELEGRAM_LIMIT, + toolRuntimeContextFromDownloads, + transcribeAudioIfNeeded, + UnifiedRunOptions +} from "./unified-ai-runner.shared"; export type {ToolCallData} from "./unified-ai-runner.shared"; export {snapshotModel, providerTargets, ollamaModelNames} from "./unified-ai-runner.shared"; @@ -137,7 +165,8 @@ async function executeUnifiedAiRequest( switch (options.provider) { case AiProvider.OPENAI: - await runOpenAi(options.msg, chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, options.msg, config, toolContext); + await runOpenAi(options.msg, chatMessages as OpenAIChatMessage[], streamMessage, controller.signal, options.stream ?? true, firstRoundStatus, options.msg, config, toolContext, + !!options.think); break; case AiProvider.OLLAMA: const currentModel = config.ollamaChatTarget.model; @@ -259,7 +288,7 @@ export async function runUnifiedAi(options: UnifiedRunOptions): Promise { aiLog("debug", "run.queue.target", {target: aiLogProviderTarget(queueTarget), cancelId: cancel.id}); try { - const queueMessage = await streamMessage.start(Environment.getAiQueueText(options.provider, 0)); + const queueMessage = await streamMessage.start(Environment.waitThinkText); setAiCancelMessageId(cancel.id, queueMessage.message_id); aiLog("info", "run.queue.enter", { cancelId: cancel.id, @@ -326,8 +355,4 @@ export async function runUnifiedAi(options: UnifiedRunOptions): Promise { aborted: controller.signal.aborted, }); } -} - -export class UnifiedAiRunner { - static run = runUnifiedAi; -} +} \ No newline at end of file diff --git a/src/commands/openai-chat.ts b/src/commands/openai-chat.ts index 24aebde..d14f12a 100644 --- a/src/commands/openai-chat.ts +++ b/src/commands/openai-chat.ts @@ -16,6 +16,6 @@ export class OpenAIChat extends ChatCommand { description = Environment.commandDescriptions.openAiChat; async execute(msg: Message, match?: RegExpExecArray): Promise { - await runUnifiedAi({provider: AiProvider.OPENAI, msg: msg, text: match?.[3] ?? "", stream: true}); + await runUnifiedAi({provider: AiProvider.OPENAI, msg: msg, text: match?.[3] ?? "", stream: true, think: true}); } } diff --git a/src/common/environment.ts b/src/common/environment.ts index fa8397b..94ca178 100644 --- a/src/common/environment.ts +++ b/src/common/environment.ts @@ -1166,6 +1166,8 @@ export class Environment { const name = isString(toolCall) ? toolCall : toolCall.name; return name === PYTHON_INTERPRETER_TOOL_NAME ? this.text("getUseToolText.python", "👨‍💻 Running `Python`") + : name === "code_interpreter" + ? this.text("getUseToolText.codeInterpreter", "👨‍💻 Running `Code Interpreter`") : this.text("getUseToolText.default", "🔧 Using tool `{name}`", {name}); }).join("\n"); } diff --git a/src/model/ai-model-capabilities.ts b/src/model/ai-model-capabilities.ts index d800240..175171d 100644 --- a/src/model/ai-model-capabilities.ts +++ b/src/model/ai-model-capabilities.ts @@ -7,6 +7,7 @@ export class AiModelCapabilities { thinking: AiCapabilityInfo | undefined; extendedThinking: AiCapabilityInfo | undefined; tools: AiCapabilityInfo | undefined; + toolRank: AiCapabilityInfo | undefined; audio: AiCapabilityInfo | undefined; documents: AiCapabilityInfo | undefined; outputImages: AiCapabilityInfo | undefined;