test support for images (ollama vision models)

src/commands/gemini-chat.ts

@@ -80,6 +80,8 @@ export class GeminiChat extends ChatCommand {
                 editFn: async (text) => {
                     await editMessageText(chatId, waitMessage.message_id, escapeMarkdownV2Text(text), "Markdown");
                 },
+                onStop: async () => {
+                }
             });
 
             try {

src/commands/ollama-chat.ts

@@ -6,12 +6,16 @@ import {
     editMessageText,
     escapeMarkdownV2Text,
     extractText,
+    getPhotoMaxSize,
     logError,
     replyToMessage,
     startIntervalEditor
 } from "../util/utils";
 import {Environment} from "../common/environment";
 import {MessageStore} from "../common/message-store";
+import axios from "axios";
+import * as fs from "node:fs";
+import path from "node:path";
 
 export class OllamaChat extends ChatCommand {
     regexp = /^\/ollama\s([^]+)/;
@@ -28,17 +32,40 @@ export class OllamaChat extends ChatCommand {
 
         const chatId = msg.chat.id;
 
+        let imageFilePath: string | null = null;
+
+        const maxSize = await getPhotoMaxSize(msg.photo, 600);
+        if (maxSize) {
+            const res = await axios.get<ArrayBuffer>(maxSize.url, {responseType: "arraybuffer"});
+            const src = Buffer.from(res.data);
+
+            const imagePath = path.join(Environment.DATA_PATH, "temp");
+            if (!fs.existsSync(imagePath)) {
+                fs.mkdirSync(imagePath);
+            }
+
+            imageFilePath = path.join(imagePath, maxSize.unique_file_id + ".jpg");
+
+            try {
+                fs.writeFileSync(imageFilePath, src);
+            } catch (e) {
+                console.error(e);
+                imageFilePath = null;
+            }
+        }
+
         const messageParts = await collectReplyChainText(msg);
         console.log("MESSAGE PARTS", messageParts);
 
-        const chatMessages = messageParts.map(part => {
+        const chatMessages = messageParts.map((part, i) => {
             return {
                 role: part.bot ? "ASSISTANT" : "USER",
-                content: extractText(part.content, Environment.BOT_PREFIX)
+                content: extractText(part.content, Environment.BOT_PREFIX),
+                images: imageFilePath && i === 0 ? [imageFilePath] : null
             };
         });
         chatMessages.reverse();
-        chatMessages.unshift({role: "SYSTEM", content: Environment.SYSTEM_PROMPT});
+        chatMessages.unshift({role: "SYSTEM", content: Environment.SYSTEM_PROMPT, images: null});
 
         let waitMessage: Message;
 
@@ -71,6 +98,8 @@ export class OllamaChat extends ChatCommand {
                 editFn: async (text) => {
                     await editMessageText(chatId, waitMessage.message_id, escapeMarkdownV2Text(text), "Markdown");
                 },
+                onStop: async () => {
+                }
             });
 
             try {
@@ -103,7 +132,7 @@ export class OllamaChat extends ChatCommand {
 
                         waitMessage.reply_to_message = msg;
                         waitMessage.text = currentText;
-                        MessageStore.put(waitMessage);
+                        await MessageStore.put(waitMessage);
 
                         await replyToMessage(waitMessage, `⏱️ ${diff}s`);
                         break;

src/commands/quote.ts

@@ -298,16 +298,16 @@ async function getBackground(
     const msgPhoto = photoArr && photoArr.length ? photoArr[photoArr.length - 1] : undefined;
 
     if (msgPhoto?.file_id) {
-        const url = await getFileUrl(bot, msgPhoto.file_id);
+        const url = await getFileUrl(msgPhoto.file_id);
         const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
         src = Buffer.from(res.data);
     } else {
         if (author.userId) {
-            src = await getUserAvatar(bot, author.userId);
+            src = await getUserAvatar(author.userId);
         } else if (author.chatId) {
-            src = await getChatAvatar(bot, author.chatId);
+            src = await getChatAvatar(author.chatId);
         } else if (!isForwarded && reply.from?.id) {
-            src = await getUserAvatar(bot, reply.from.id);
+            src = await getUserAvatar(reply.from.id);
         }
     }
 

src/commands/system-specs.ts

@@ -1,14 +1,14 @@
 import {ChatCommand} from "../base/chat-command";
 import {logError, oldSendMessage} from "../util/utils";
 import {Message} from "typescript-telegram-bot-api";
-import {systemSpecsText} from "../index";
+import {systemInfoText} from "../index";
 
 export class SystemSpecs implements ChatCommand {
-    regexp = /^\/systemspecs/i;
+    regexp = /^\/systeminfo/i;
-    title = "/systemSpecs";
+    title = "/systemInfo";
-    description = "System specifications of system";
+    description = "System information";
 
     async execute(msg: Message) {
-        await oldSendMessage(msg, systemSpecsText).catch(logError);
+        await oldSendMessage(msg, systemInfoText).catch(logError);
     }
 }

src/db/message-dao.ts

@@ -5,7 +5,8 @@ import {and, eq} from "drizzle-orm";
 import {inArray} from "drizzle-orm/sql/expressions/conditions";
 import {Message} from "typescript-telegram-bot-api";
 import {Dao} from "../base/dao";
-import {buildExcludedSet} from "../util/utils";
+import {buildExcludedSet, extractTextMessage} from "../util/utils";
+import {Environment} from "../common/environment";
 
 export class MessageDao extends Dao<StoredMessage> {
 
@@ -88,7 +89,7 @@ export class MessageDao extends Dao<StoredMessage> {
                 id: msg.message_id,
                 replyToMessageId: msg.reply_to_message?.message_id,
                 fromId: msg.from.id,
-                text: msg.text,
+                text: extractTextMessage(msg, Environment.BOT_PREFIX),
                 date: msg.date,
                 firstName: msg.from.first_name,
                 lastName: msg.from.last_name,

src/index.ts

@@ -72,10 +72,10 @@ export const ollama = new Ollama({
 
 export const googleAi = new GoogleGenAI({apiKey: Environment.GEMINI_API_KEY});
 
-export let systemSpecsText: string = "";
+export let systemInfoText: string = "";
 
-export function setSystemSpecs(systemSpecs: string) {
+export function setSystemInfo(info: string) {
-    systemSpecsText = systemSpecs;
+    systemInfoText = info;
 }
 
 export const chatCommands: ChatCommand[] = [
@@ -149,22 +149,6 @@ bot.on("my_chat_member", async (u) => {
     console.log("my_chat_member", u);
 });
 
-bot.on("message", async (message) => {
-    console.log("message", message);
-
-    await UserStore.put(message.from);
-
-    if ((message.new_chat_members?.length || 0 > 0)) {
-        await bot.sendMessage({chat_id: message.chat.id, text: randomValue(inviteAnswers)}).catch(logError);
-        return;
-    }
-
-    if (message.left_chat_member && message.left_chat_member.id !== botUser.id) {
-        await bot.sendMessage({chat_id: message.chat.id, text: randomValue(kickAnswers)}).catch(logError);
-        return;
-    }
-});
-
 bot.on("edited_message", async (msg) => {
     console.log("edited_message", msg);
 
@@ -175,8 +159,20 @@ bot.on("edited_message", async (msg) => {
     await MessageStore.put(msg);
 });
 
-bot.on("message:text", async (msg) => {
+bot.on("message", async (msg) => {
-    await MessageStore.put(msg);
+    console.log("message", msg);
+
+    await Promise.all([MessageStore.put(msg), UserStore.put(msg.from)]);
+
+    if ((msg.new_chat_members?.length || 0 > 0)) {
+        await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(inviteAnswers)}).catch(logError);
+        return;
+    }
+
+    if (msg.left_chat_member && msg.left_chat_member.id !== botUser.id) {
+        await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(kickAnswers)}).catch(logError);
+        return;
+    }
 
     if (muted.has(msg.from.id)) return;
 

src/util/utils.ts

@@ -1,18 +1,10 @@
 import * as si from "systeminformation";
 import {ChatCommand} from "../base/chat-command";
 import {CallbackCommand} from "../base/callback-command";
-import {
+import {CallbackQuery, InlineKeyboardMarkup, Message, ParseMode, PhotoSize, User} from "typescript-telegram-bot-api";
-    CallbackQuery,
-    InlineKeyboardMarkup,
-    Message,
-    ParseMode,
-    PhotoSize,
-    TelegramBot,
-    User
-} from "typescript-telegram-bot-api";
 import {Environment} from "../common/environment";
 import {TelegramError} from "typescript-telegram-bot-api/dist/errors";
-import {bot, botUser, messageDao, setSystemSpecs} from "../index";
+import {bot, botUser, messageDao, setSystemInfo} from "../index";
 import os from "os";
 import axios from "axios";
 import {MessagePart} from "../common/message-part";
@@ -240,7 +232,7 @@ export async function initSystemSpecs(): Promise<void> {
             `CPU: ${cpu.manufacturer} ${cpu.brand} ${cpu.physicalCores} cores ${cpu.cores} threads\n` +
             `RAM: ${ramSize} GB`;
 
-        setSystemSpecs(text);
+        setSystemInfo(text);
         return Promise.resolve();
     } catch (e) {
         return Promise.reject(e);
@@ -335,18 +327,18 @@ export function escapeMarkdownV2Text(s: string) {
     return s;
 }
 
-export async function getFileUrl(bot: TelegramBot, fileId: string) {
+export async function getFileUrl(fileId: string): Promise<string> {
     const file = await bot.getFile({file_id: fileId});
     return `https://api.telegram.org/file/bot${bot.botToken}/${file.file_path}`;
 }
 
-export async function getChatAvatar(bot: TelegramBot, chatId: number): Promise<Buffer | null> {
+export async function getChatAvatar(chatId: number): Promise<Buffer | null> {
     try {
         const chat = await bot.getChat({chat_id: chatId});
         const photo = chat?.photo?.big_file_id || chat?.photo?.small_file_id;
         if (!photo) return null;
 
-        const url = await getFileUrl(bot, photo);
+        const url = await getFileUrl(photo);
         const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
         return Buffer.from(res.data);
     } catch {
@@ -354,12 +346,12 @@ export async function getChatAvatar(bot: TelegramBot, chatId: number): Promise<B
     }
 }
 
-export async function getUserAvatar(bot: TelegramBot, userId: number): Promise<Buffer | null> {
+export async function getUserAvatar(userId: number): Promise<Buffer | null> {
     const photos = await bot.getUserProfilePhotos({user_id: userId, limit: 1});
     const last: PhotoSize | undefined = photos.photos?.[0]?.[photos.photos[0].length - 1];
     if (!last) return null;
 
-    const url = await getFileUrl(bot, last.file_id);
+    const url = await getFileUrl(last.file_id);
     const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
     return Buffer.from(res.data);
 }
@@ -401,7 +393,7 @@ export async function collectReplyChainText(triggerMsg: Message, prefix: string
         const t = extractTextMessage(triggerMsg, prefix);
         if (t) parts.push({
             bot: triggerMsg.from.id === botUser.id,
-            content: triggerMsg.text,
+            content: t,
             name: triggerMsg.from.first_name
         });
     }
@@ -499,7 +491,7 @@ export async function waveDistortSharp(
 
             if (sx < 0 || sx >= width || sy < 0 || sy >= height) {
                 // прозрачный пиксель
-                out[di + 0] = 0;
+                out[di] = 0;
                 out[di + 1] = 0;
                 out[di + 2] = 0;
                 out[di + 3] = 0;
@@ -678,6 +670,7 @@ export function startIntervalEditor(params: {
     intervalMs: number;
     getText: () => string;
     editFn: (text: string) => Promise<void>;
+    onStop: () => Promise<void>;
 }) {
     let lastSent = "";
     let stopped = false;
@@ -706,6 +699,7 @@ export function startIntervalEditor(params: {
             stopped = true;
             clearInterval(timer);
             await tick();
+            await params.onStop();
         },
     };
 }
@@ -757,3 +751,36 @@ export function getRuntimeInfo(): RuntimeInfo {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     return {runtime: "unknown", version: String((process as any).version ?? "")};
 }
+
+export type PhotoMaxSize = { width: number, height: number, url: string; unique_file_id: string; };
+
+export async function getPhotoMaxSize(photos: PhotoSize[], target: number = 1280): Promise<PhotoMaxSize | null> {
+    if (!photos) return null;
+
+    photos = photos.filter(p => Math.max(p.width, p.height) <= target);
+
+    if (photos.length === 0) return null;
+
+    if (photos.length === 1) {
+        return mapPhotoSizeToMax(photos[0]);
+    }
+
+    const max = photos.reduce((prev, cur) => {
+        if (!prev) return cur;
+
+        return cur.width * cur.height > prev.width * prev.height ? cur : prev;
+    }, null);
+
+    if (!max) return null;
+    return mapPhotoSizeToMax(max);
+}
+
+export async function mapPhotoSizeToMax(size: PhotoSize): Promise<PhotoMaxSize | null> {
+    if (!size) return null;
+    return {
+        width: size.width,
+        height: size.height,
+        url: await getFileUrl(size.file_id),
+        unique_file_id: size.file_unique_id
+    };
+}