Add unified request pipeline stages
This commit is contained in:
+47
-114
@@ -1,32 +1,13 @@
|
||||
import test, {after} from "node:test";
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tg-chat-bot-rag-"));
|
||||
process.env.BOT_TOKEN = process.env.BOT_TOKEN ?? "test-token";
|
||||
process.env.CREATOR_ID = process.env.CREATOR_ID ?? "1";
|
||||
process.env.DATA_PATH = tempRoot;
|
||||
process.env.DB_PATH = `file:${path.join(tempRoot, "test.sqlite")}`;
|
||||
process.env.TEST_ENVIRONMENT = "true";
|
||||
|
||||
const {Environment} = await import("../dist/common/environment.js");
|
||||
Environment.load();
|
||||
|
||||
const {DatabaseManager} = await import("../dist/db/database-manager.js");
|
||||
DatabaseManager.init();
|
||||
await DatabaseManager.ready;
|
||||
|
||||
const {ArtifactStore} = await import("../dist/common/artifact-store.js");
|
||||
const {filterUserVisibleStoredAttachments} = await import("../dist/common/stored-attachment-utils.js");
|
||||
const {
|
||||
buildRagArtifactPayload,
|
||||
} = await import("../dist/ai/rag-artifact-payload.js");
|
||||
const {
|
||||
filterUserVisibleStoredAttachments,
|
||||
} = await import("../dist/common/attachment-visibility.js");
|
||||
const {AiProvider} = await import("../dist/model/ai-provider.js");
|
||||
const {persistRagArtifactAttachment} = await import("../dist/ai/rag-artifact-store.js");
|
||||
|
||||
after(async () => {
|
||||
await DatabaseManager.close().catch(() => undefined);
|
||||
fs.rmSync(tempRoot, {recursive: true, force: true});
|
||||
});
|
||||
|
||||
test("internal artifacts are not treated as user-visible attachments", () => {
|
||||
const visible = filterUserVisibleStoredAttachments([
|
||||
@@ -50,105 +31,57 @@ test("internal artifacts are not treated as user-visible attachments", () => {
|
||||
assert.equal(visible[0].fileId, "visible");
|
||||
});
|
||||
|
||||
test("RAG artifacts persist structured ollama metadata", async () => {
|
||||
const chatId = 42;
|
||||
const messageId = 7;
|
||||
|
||||
const attachment = await persistRagArtifactAttachment({
|
||||
test("RAG artifact payload keeps ollama retrieval metadata", () => {
|
||||
const payload = buildRagArtifactPayload({
|
||||
provider: AiProvider.OLLAMA,
|
||||
prepared: {
|
||||
provider: AiProvider.OLLAMA,
|
||||
prepared: true,
|
||||
cleanup: async () => undefined,
|
||||
artifact: {
|
||||
query: "What is in the file?",
|
||||
extractedDocuments: [
|
||||
{documentIndex: 0, fileName: "report.txt", textChars: 120},
|
||||
],
|
||||
selectedChunks: [
|
||||
{
|
||||
sourceId: "doc1-1",
|
||||
documentIndex: 0,
|
||||
documentName: "report.txt",
|
||||
chunkIndex: 0,
|
||||
chunkCount: 1,
|
||||
textChars: 120,
|
||||
score: 0.91,
|
||||
},
|
||||
],
|
||||
skippedDocuments: [
|
||||
{documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
|
||||
],
|
||||
providerState: {
|
||||
embeddingModel: "nomic-embed-text:latest",
|
||||
topK: 8,
|
||||
chunkSize: 1400,
|
||||
chunkOverlap: 220,
|
||||
maxContextChars: 14000,
|
||||
minScore: 0.12,
|
||||
maxArchiveFiles: 200,
|
||||
maxArchiveBytes: 50 * 1024 * 1024,
|
||||
maxArchiveDepth: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
downloads: [{
|
||||
kind: "document",
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
sources: [{
|
||||
fileId: "file-1",
|
||||
fileName: "report.txt",
|
||||
buffer: Buffer.from("hello world"),
|
||||
path: path.join(tempRoot, "report.txt"),
|
||||
mimeType: "text/plain",
|
||||
sizeBytes: 12,
|
||||
sha256: "abc123",
|
||||
uploadedFileId: "uploaded-1",
|
||||
}],
|
||||
chatId,
|
||||
messageId,
|
||||
details: {
|
||||
providerState: {
|
||||
provider: AiProvider.OLLAMA,
|
||||
prepared: true,
|
||||
embeddingModel: "nomic-embed-text:latest",
|
||||
topK: 8,
|
||||
chunkSize: 1400,
|
||||
chunkOverlap: 220,
|
||||
maxContextChars: 14000,
|
||||
artifact: {
|
||||
query: "What is in the file?",
|
||||
extractedDocuments: [
|
||||
{documentIndex: 0, fileName: "report.txt", textChars: 120},
|
||||
],
|
||||
selectedChunks: [
|
||||
{
|
||||
sourceId: "doc1-1",
|
||||
documentIndex: 0,
|
||||
documentName: "report.txt",
|
||||
chunkIndex: 0,
|
||||
chunkCount: 1,
|
||||
textChars: 120,
|
||||
score: 0.91,
|
||||
},
|
||||
],
|
||||
skippedDocuments: [
|
||||
{documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
|
||||
],
|
||||
providerState: {
|
||||
embeddingModel: "nomic-embed-text:latest",
|
||||
topK: 8,
|
||||
chunkSize: 1400,
|
||||
chunkOverlap: 220,
|
||||
maxContextChars: 14000,
|
||||
minScore: 0.12,
|
||||
maxArchiveFiles: 200,
|
||||
maxArchiveBytes: 50 * 1024 * 1024,
|
||||
maxArchiveDepth: 2,
|
||||
extractedDocuments: [
|
||||
{documentIndex: 0, fileName: "report.txt", textChars: 120},
|
||||
],
|
||||
selectedChunks: [
|
||||
{
|
||||
sourceId: "doc1-1",
|
||||
documentIndex: 0,
|
||||
documentName: "report.txt",
|
||||
chunkIndex: 0,
|
||||
chunkCount: 1,
|
||||
textChars: 120,
|
||||
score: 0.91,
|
||||
},
|
||||
},
|
||||
],
|
||||
skippedDocuments: [
|
||||
{documentIndex: 1, fileName: "ignored.bin", reason: "unsupported format"},
|
||||
],
|
||||
minScore: 0.12,
|
||||
maxArchiveFiles: 200,
|
||||
maxArchiveBytes: 50 * 1024 * 1024,
|
||||
maxArchiveDepth: 2,
|
||||
query: "What is in the file?",
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(attachment?.artifactKind, "rag");
|
||||
assert.equal(fs.existsSync(attachment.cachePath), true);
|
||||
|
||||
const stored = await ArtifactStore.getByMessage(chatId, messageId);
|
||||
assert.equal(stored.length, 1);
|
||||
assert.equal(stored[0].kind, "rag");
|
||||
assert.equal(stored[0].payload.providerState.query, "What is in the file?");
|
||||
assert.equal(stored[0].payload.providerState.selectedChunks[0].score, 0.91);
|
||||
assert.equal(stored[0].payload.providerState.skippedDocuments[0].reason, "unsupported format");
|
||||
assert.equal(stored[0].payload.providerState.ollama.embeddingModel, "nomic-embed-text:latest");
|
||||
assert.equal(payload.artifactKind, "rag");
|
||||
assert.equal(payload.provider, AiProvider.OLLAMA);
|
||||
assert.equal(payload.sources[0].uploadedFileId, "uploaded-1");
|
||||
assert.equal(payload.providerState.provider, AiProvider.OLLAMA);
|
||||
assert.equal(payload.providerState.query, "What is in the file?");
|
||||
assert.equal(payload.providerState.selectedChunks[0].score, 0.91);
|
||||
assert.equal(payload.providerState.skippedDocuments[0].reason, "unsupported format");
|
||||
assert.equal(payload.providerState.embeddingModel, "nomic-embed-text:latest");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user