Add unified request pipeline stages

2026-05-18 15:45:39 +03:00
parent 8cff086a8e
commit 8aede4b053
18 changed files with 905 additions and 509 deletions
@@ -0,0 +1,83 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const {
+    extractOpenAiToolCalls,
+    extractOpenAiStreamingToolCalls,
+    extractOpenAiTextDelta,
+    extractMistralToolCalls,
+    extractMistralTextDelta,
+    extractOllamaToolCalls,
+    extractOllamaTextDelta,
+} = await import("../dist/ai/provider-adapter-contract.js");
+
+test("openai contract extracts text delta and function calls", () => {
+    assert.equal(extractOpenAiTextDelta({type: "response.output_text.delta", delta: "hello"}), "hello");
+
+    const calls = extractOpenAiToolCalls({
+        output: [{
+            type: "function_call",
+            call_id: "call-1",
+            name: "read_file",
+            arguments: "{\"path\":\"src/index.ts\"}",
+        }],
+    });
+
+    assert.equal(calls.length, 1);
+    assert.equal(calls[0].id, "call-1");
+    assert.equal(calls[0].name, "read_file");
+
+    const streamed = extractOpenAiStreamingToolCalls({
+        type: "response.output_item.added",
+        item: {
+            type: "function_call",
+            id: "call-2",
+            name: "search_files",
+            arguments: "{\"query\":\"sendMessage\"}",
+        },
+    });
+
+    assert.equal(streamed.length, 1);
+    assert.equal(streamed[0].id, "call-2");
+    assert.equal(streamed[0].name, "search_files");
+});
+
+test("mistral contract extracts content and tool calls", () => {
+    assert.equal(extractMistralTextDelta({
+        content: [{text: "hello"}, {text: " world"}],
+    }), "hello world");
+
+    const calls = extractMistralToolCalls({
+        toolCalls: [{
+            id: "m-1",
+            function: {
+                name: "get_weather",
+                arguments: {location: "Moscow"},
+            },
+        }],
+    });
+
+    assert.equal(calls.length, 1);
+    assert.equal(calls[0].id, "m-1");
+    assert.equal(calls[0].name, "get_weather");
+});
+
+test("ollama contract extracts content and tool calls", () => {
+    assert.equal(extractOllamaTextDelta({
+        message: {content: "hello from ollama"},
+    }), "hello from ollama");
+
+    const calls = extractOllamaToolCalls({
+        tool_calls: [{
+            id: "o-1",
+            function: {
+                name: "web_search",
+                arguments: {query: "openai docs"},
+            },
+        }],
+    });
+
+    assert.equal(calls.length, 1);
+    assert.equal(calls[0].id, "o-1");
+    assert.equal(calls[0].name, "web_search");
+});
@@ -1,32 +1,13 @@
-import test, {after} from "node:test";
+import test from "node:test";
 import assert from "node:assert/strict";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";

-const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tg-chat-bot-rag-"));
-process.env.BOT_TOKEN = process.env.BOT_TOKEN ?? "test-token";
-process.env.CREATOR_ID = process.env.CREATOR_ID ?? "1";
-process.env.DATA_PATH = tempRoot;
-process.env.DB_PATH = `file:${path.join(tempRoot, "test.sqlite")}`;
-process.env.TEST_ENVIRONMENT = "true";
-
-const {Environment} = await import("../dist/common/environment.js");
-Environment.load();
-
-const {DatabaseManager} = await import("../dist/db/database-manager.js");
-DatabaseManager.init();
-await DatabaseManager.ready;
-
-const {ArtifactStore} = await import("../dist/common/artifact-store.js");
-const {filterUserVisibleStoredAttachments} = await import("../dist/common/stored-attachment-utils.js");
+const {
+    buildRagArtifactPayload,
+} = await import("../dist/ai/rag-artifact-payload.js");
+const {
+    filterUserVisibleStoredAttachments,
+} = await import("../dist/common/attachment-visibility.js");
 const {AiProvider} = await import("../dist/model/ai-provider.js");
-const {persistRagArtifactAttachment} = await import("../dist/ai/rag-artifact-store.js");
-
-after(async () => {
-    await DatabaseManager.close().catch(() => undefined);
-    fs.rmSync(tempRoot, {recursive: true, force: true});
-});

 test("internal artifacts are not treated as user-visible attachments", () => {
    const visible = filterUserVisibleStoredAttachments([
@@ -50,105 +31,57 @@ test("internal artifacts are not treated as user-visible attachments", () => {
    assert.equal(visible[0].fileId, "visible");
 });

-test("RAG artifacts persist structured ollama metadata", async () => {
-    const chatId = 42;
-    const messageId = 7;
-
-    const attachment = await persistRagArtifactAttachment({
+test("RAG artifact payload keeps ollama retrieval metadata", () => {
+    const payload = buildRagArtifactPayload({
        provider: AiProvider.OLLAMA,
-        prepared: {
-            provider: AiProvider.OLLAMA,
-            prepared: true,
-            cleanup: async () => undefined,
-            artifact: {
-                query: "What is in the file?",
-                extractedDocuments: [
-                    {documentIndex: 0, fileName: "report.txt", textChars: 120},
-                ],
-                selectedChunks: [
-                    {
-                        sourceId: "doc1-1",
-                        documentIndex: 0,
-                        documentName: "report.txt",
-                        chunkIndex: 0,
-                        chunkCount: 1,
-                        textChars: 120,
-                        score: 0.91,
-                    },
-                ],
-                skippedDocuments: [
-                    {documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
-                ],
-                providerState: {
-                    embeddingModel: "nomic-embed-text:latest",
-                    topK: 8,
-                    chunkSize: 1400,
-                    chunkOverlap: 220,
-                    maxContextChars: 14000,
-                    minScore: 0.12,
-                    maxArchiveFiles: 200,
-                    maxArchiveBytes: 50 * 1024 * 1024,
-                    maxArchiveDepth: 2,
-                },
-            },
-        },
-        downloads: [{
-            kind: "document",
+        createdAt: "2026-01-01T00:00:00.000Z",
+        sources: [{
            fileId: "file-1",
            fileName: "report.txt",
-            buffer: Buffer.from("hello world"),
-            path: path.join(tempRoot, "report.txt"),
+            mimeType: "text/plain",
+            sizeBytes: 12,
+            sha256: "abc123",
+            uploadedFileId: "uploaded-1",
        }],
-        chatId,
-        messageId,
-        details: {
+        providerState: {
+            provider: AiProvider.OLLAMA,
+            prepared: true,
            embeddingModel: "nomic-embed-text:latest",
            topK: 8,
            chunkSize: 1400,
            chunkOverlap: 220,
            maxContextChars: 14000,
-            artifact: {
-                query: "What is in the file?",
-                extractedDocuments: [
-                    {documentIndex: 0, fileName: "report.txt", textChars: 120},
-                ],
-                selectedChunks: [
-                    {
-                        sourceId: "doc1-1",
-                        documentIndex: 0,
-                        documentName: "report.txt",
-                        chunkIndex: 0,
-                        chunkCount: 1,
-                        textChars: 120,
-                        score: 0.91,
-                    },
-                ],
-                skippedDocuments: [
-                    {documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
-                ],
-                providerState: {
-                    embeddingModel: "nomic-embed-text:latest",
-                    topK: 8,
-                    chunkSize: 1400,
-                    chunkOverlap: 220,
-                    maxContextChars: 14000,
-                    minScore: 0.12,
-                    maxArchiveFiles: 200,
-                    maxArchiveBytes: 50 * 1024 * 1024,
-                    maxArchiveDepth: 2,
+            extractedDocuments: [
+                {documentIndex: 0, fileName: "report.txt", textChars: 120},
+            ],
+            selectedChunks: [
+                {
+                    sourceId: "doc1-1",
+                    documentIndex: 0,
+                    documentName: "report.txt",
+                    chunkIndex: 0,
+                    chunkCount: 1,
+                    textChars: 120,
+                    score: 0.91,
                },
-            },
+            ],
+            skippedDocuments: [
+                {documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
+            ],
+            minScore: 0.12,
+            maxArchiveFiles: 200,
+            maxArchiveBytes: 50 * 1024 * 1024,
+            maxArchiveDepth: 2,
+            query: "What is in the file?",
        },
    });

-    assert.equal(attachment?.artifactKind, "rag");
-    assert.equal(fs.existsSync(attachment.cachePath), true);
-
-    const stored = await ArtifactStore.getByMessage(chatId, messageId);
-    assert.equal(stored.length, 1);
-    assert.equal(stored[0].kind, "rag");
-    assert.equal(stored[0].payload.providerState.query, "What is in the file?");
-    assert.equal(stored[0].payload.providerState.selectedChunks[0].score, 0.91);
-    assert.equal(stored[0].payload.providerState.skippedDocuments[0].reason, "unsupported format");
-    assert.equal(stored[0].payload.providerState.ollama.embeddingModel, "nomic-embed-text:latest");
+    assert.equal(payload.artifactKind, "rag");
+    assert.equal(payload.provider, AiProvider.OLLAMA);
+    assert.equal(payload.sources[0].uploadedFileId, "uploaded-1");
+    assert.equal(payload.providerState.provider, AiProvider.OLLAMA);
+    assert.equal(payload.providerState.query, "What is in the file?");
+    assert.equal(payload.providerState.selectedChunks[0].score, 0.91);
+    assert.equal(payload.providerState.skippedDocuments[0].reason, "unsupported format");
+    assert.equal(payload.providerState.embeddingModel, "nomic-embed-text:latest");
 });