shitton of the ai changes
This commit is contained in:
@@ -0,0 +1,319 @@
|
||||
import {AiProvider} from "../model/ai-provider";
|
||||
import {AI_VOICE_MODE_TRANSCRIPT, DEFAULT_AI_RESPONSE_LANGUAGE} from "../common/user-ai-settings";
|
||||
import {Environment} from "../common/environment";
|
||||
import {UserRequestPipeline, type UserRequestPipelineState, type UserRequestPipelineStage} from "./user-request-pipeline";
|
||||
import type {AiDownloadedFile} from "./telegram-attachments";
|
||||
import type {TelegramStreamMessage} from "./telegram-stream-message";
|
||||
import type {ChatMessage} from "./chat-messages-types";
|
||||
import type {OpenAIChatMessage} from "./openai-chat-message";
|
||||
import type {MistralChatMessage} from "./mistral-chat-message";
|
||||
import type {PreparedDocumentRag} from "./document-rag-pipeline";
|
||||
import {prepareDocumentRag} from "./document-rag-pipeline";
|
||||
import {persistRagArtifactAttachment} from "./rag-artifact-store";
|
||||
import {persistTranscriptArtifactAttachment} from "./transcript-artifact-store";
|
||||
import type {ToolRuntimeContext} from "./tools/runtime";
|
||||
import {
|
||||
appendTranscriptToChatMessages,
|
||||
collectTextMessages,
|
||||
initialStatus,
|
||||
providerName,
|
||||
RuntimeConfigSnapshot,
|
||||
stripAudioFromRunnerMessages,
|
||||
toolRuntimeContextFromDownloads,
|
||||
transcribeAudioIfNeeded,
|
||||
UnifiedRunOptions,
|
||||
} from "./unified-ai-runner.shared";
|
||||
import {aiLog} from "../logging/ai-logger";
|
||||
import {isTranscribableAudioDownload} from "./speech-to-text";
|
||||
|
||||
export type PreparedUnifiedAiRequest = {
|
||||
chatMessages: Array<OpenAIChatMessage | MistralChatMessage | ChatMessage>;
|
||||
imageCount: number;
|
||||
firstRoundStatus: string;
|
||||
toolContext: ToolRuntimeContext;
|
||||
preparedDocumentRag?: PreparedDocumentRag;
|
||||
finishAfterTranscript: boolean;
|
||||
cleanup: () => Promise<void>;
|
||||
};
|
||||
|
||||
type MutablePreparedContext = {
|
||||
chatMessages: Array<OpenAIChatMessage | MistralChatMessage | ChatMessage>;
|
||||
imageCount: number;
|
||||
firstRoundStatus: string;
|
||||
toolContext: ToolRuntimeContext;
|
||||
transcript: string;
|
||||
preparedDocumentRag?: PreparedDocumentRag;
|
||||
finishAfterTranscript: boolean;
|
||||
};
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function runtimeTargetFor(options: UnifiedRunOptions, config: RuntimeConfigSnapshot) {
|
||||
return options.provider === AiProvider.OLLAMA
|
||||
? config.ollamaChatTarget
|
||||
: options.provider === AiProvider.MISTRAL
|
||||
? config.mistralChatTarget
|
||||
: config.openAiChatTarget;
|
||||
}
|
||||
|
||||
function createAiRequestPipelineState(options: UnifiedRunOptions): UserRequestPipelineState {
|
||||
return {
|
||||
requestId: `ai:${options.msg.chat.id}:${options.msg.message_id}:${Date.now()}`,
|
||||
chatId: options.msg.chat.id,
|
||||
messageId: options.msg.message_id,
|
||||
replyToMessageId: options.msg.reply_to_message?.message_id,
|
||||
fromId: options.msg.from?.id ?? 0,
|
||||
receivedAt: nowIso(),
|
||||
text: options.text,
|
||||
settings: {
|
||||
provider: options.provider,
|
||||
responseLanguage: options.responseLanguage ?? DEFAULT_AI_RESPONSE_LANGUAGE,
|
||||
contextSize: options.contextSize,
|
||||
voiceMode: options.voiceMode ?? "execute",
|
||||
imageOutputMode: "photo",
|
||||
},
|
||||
inputAttachments: [],
|
||||
outputAttachments: [],
|
||||
artifacts: [],
|
||||
toolRankDecisions: [],
|
||||
audit: [],
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareUnifiedAiRequestPipeline(params: {
|
||||
options: UnifiedRunOptions;
|
||||
config: RuntimeConfigSnapshot;
|
||||
downloads: AiDownloadedFile[];
|
||||
streamMessage: TelegramStreamMessage;
|
||||
controller: AbortController;
|
||||
}): Promise<PreparedUnifiedAiRequest> {
|
||||
const {options, config, downloads, streamMessage, controller} = params;
|
||||
const prepared: MutablePreparedContext = {
|
||||
chatMessages: [],
|
||||
imageCount: 0,
|
||||
firstRoundStatus: Environment.waitThinkText,
|
||||
toolContext: {},
|
||||
transcript: "",
|
||||
finishAfterTranscript: false,
|
||||
};
|
||||
|
||||
const stages: UserRequestPipelineStage[] = [
|
||||
{
|
||||
name: "audit_start",
|
||||
async run() {
|
||||
return {
|
||||
stage: "audit_start",
|
||||
status: "succeeded",
|
||||
details: {
|
||||
phase: "ai_request_prepare",
|
||||
provider: options.provider,
|
||||
downloads: downloads.map(download => ({
|
||||
kind: download.kind,
|
||||
fileName: download.fileName,
|
||||
mimeType: download.mimeType,
|
||||
sizeBytes: download.sizeBytes ?? download.buffer.length,
|
||||
})),
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collect_conversation_context",
|
||||
async run() {
|
||||
const collected = await collectTextMessages(
|
||||
options.msg,
|
||||
options.text,
|
||||
options.provider,
|
||||
downloads,
|
||||
config,
|
||||
runtimeTargetFor(options, config),
|
||||
options.responseLanguage ?? DEFAULT_AI_RESPONSE_LANGUAGE,
|
||||
);
|
||||
prepared.chatMessages = collected.chatMessages as typeof prepared.chatMessages;
|
||||
prepared.imageCount = collected.imageCount;
|
||||
prepared.firstRoundStatus = initialStatus(downloads, prepared.imageCount);
|
||||
prepared.toolContext = toolRuntimeContextFromDownloads(downloads);
|
||||
|
||||
return {
|
||||
stage: "collect_conversation_context",
|
||||
status: "succeeded",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prepare_text_context",
|
||||
async run() {
|
||||
streamMessage.setStatus(prepared.firstRoundStatus);
|
||||
await streamMessage.flush();
|
||||
|
||||
return {
|
||||
stage: "prepare_text_context",
|
||||
status: "succeeded",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "resolve_runtime",
|
||||
async run() {
|
||||
return {
|
||||
stage: "resolve_runtime",
|
||||
status: "succeeded",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "speech_to_text",
|
||||
async run() {
|
||||
prepared.transcript = await transcribeAudioIfNeeded(
|
||||
options.provider,
|
||||
options.msg.from?.id,
|
||||
downloads,
|
||||
streamMessage,
|
||||
controller.signal,
|
||||
).catch(error => {
|
||||
if (downloads.some(isTranscribableAudioDownload)) throw error;
|
||||
return "";
|
||||
});
|
||||
|
||||
const transcript = prepared.transcript.trim();
|
||||
if (!transcript) {
|
||||
return {
|
||||
stage: "speech_to_text",
|
||||
status: "skipped",
|
||||
};
|
||||
}
|
||||
|
||||
const transcriptArtifact = await persistTranscriptArtifactAttachment({
|
||||
provider: options.provider,
|
||||
transcript,
|
||||
downloads,
|
||||
chatId: options.msg.chat.id,
|
||||
messageId: options.msg.message_id,
|
||||
});
|
||||
if (transcriptArtifact) {
|
||||
await streamMessage.storeInternalAttachment(transcriptArtifact);
|
||||
}
|
||||
|
||||
if (options.voiceMode === AI_VOICE_MODE_TRANSCRIPT) {
|
||||
prepared.finishAfterTranscript = true;
|
||||
streamMessage.replaceText(`[Расшифровка]\n${transcript}`);
|
||||
await streamMessage.finish();
|
||||
return {
|
||||
stage: "speech_to_text",
|
||||
status: "succeeded",
|
||||
fallbackAction: "continue_without_stage",
|
||||
};
|
||||
}
|
||||
|
||||
appendTranscriptToChatMessages(prepared.chatMessages, transcript);
|
||||
stripAudioFromRunnerMessages(prepared.chatMessages);
|
||||
aiLog("debug", "request.transcript.appended", {
|
||||
provider: providerName(options.provider),
|
||||
transcriptChars: transcript.length,
|
||||
chatMessages: prepared.chatMessages.length,
|
||||
});
|
||||
|
||||
return {
|
||||
stage: "speech_to_text",
|
||||
status: "succeeded",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "document_rag",
|
||||
async run() {
|
||||
if (prepared.finishAfterTranscript) {
|
||||
return {
|
||||
stage: "document_rag",
|
||||
status: "skipped",
|
||||
};
|
||||
}
|
||||
|
||||
prepared.preparedDocumentRag = await prepareDocumentRag(
|
||||
options.provider,
|
||||
downloads,
|
||||
prepared.chatMessages,
|
||||
streamMessage,
|
||||
config,
|
||||
controller.signal,
|
||||
options.text,
|
||||
);
|
||||
|
||||
const ragArtifact = await persistRagArtifactAttachment({
|
||||
provider: options.provider,
|
||||
prepared: prepared.preparedDocumentRag,
|
||||
downloads,
|
||||
chatId: options.msg.chat.id,
|
||||
messageId: options.msg.message_id,
|
||||
details: prepared.preparedDocumentRag?.provider === AiProvider.OPENAI
|
||||
? {uploadedFileIds: prepared.preparedDocumentRag.uploadedFileIds}
|
||||
: prepared.preparedDocumentRag?.provider === AiProvider.OLLAMA
|
||||
? {
|
||||
embeddingModel: config.ollamaDocumentsTarget.model,
|
||||
topK: config.ollamaRagTopK,
|
||||
chunkSize: config.ollamaRagChunkSize,
|
||||
chunkOverlap: config.ollamaRagChunkOverlap,
|
||||
maxContextChars: config.ollamaRagMaxContextChars,
|
||||
artifact: prepared.preparedDocumentRag.artifact,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
if (ragArtifact) {
|
||||
await streamMessage.storeInternalAttachment(ragArtifact);
|
||||
}
|
||||
|
||||
return {
|
||||
stage: "document_rag",
|
||||
status: prepared.preparedDocumentRag ? "succeeded" : "skipped",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "audit_finish",
|
||||
async run() {
|
||||
return {
|
||||
stage: "audit_finish",
|
||||
status: "succeeded",
|
||||
details: {
|
||||
phase: "ai_request_prepare",
|
||||
chatMessages: prepared.chatMessages.length,
|
||||
imageCount: prepared.imageCount,
|
||||
hasTranscript: !!prepared.transcript.trim(),
|
||||
hasDocumentRag: !!prepared.preparedDocumentRag,
|
||||
finishAfterTranscript: prepared.finishAfterTranscript,
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const state = createAiRequestPipelineState(options);
|
||||
const pipeline = new UserRequestPipeline({
|
||||
stages,
|
||||
stageNames: [
|
||||
"audit_start",
|
||||
"collect_conversation_context",
|
||||
"prepare_text_context",
|
||||
"resolve_runtime",
|
||||
"speech_to_text",
|
||||
"document_rag",
|
||||
"audit_finish",
|
||||
],
|
||||
});
|
||||
await pipeline.run(state, controller.signal);
|
||||
await streamMessage.storePipelineAudit(state.audit);
|
||||
|
||||
return {
|
||||
chatMessages: prepared.chatMessages,
|
||||
imageCount: prepared.imageCount,
|
||||
firstRoundStatus: prepared.firstRoundStatus,
|
||||
toolContext: prepared.toolContext,
|
||||
preparedDocumentRag: prepared.preparedDocumentRag,
|
||||
finishAfterTranscript: prepared.finishAfterTranscript,
|
||||
cleanup: async () => {
|
||||
await prepared.preparedDocumentRag?.cleanup();
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user