melod1n/tg-chat-bot
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

test support for images (ollama vision models)

Browse Source
This commit is contained in:
Danil Nikolaev
2026-01-12 19:19:08 +03:00
parent 62cab41b5b
commit 6eff5d17ea
7 changed files with 112 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/commands/gemini-chat.ts
+2
View File
@@ -80,6 +80,8 @@ export class GeminiChat extends ChatCommand {
editFn: async (text) => {
await editMessageText(chatId, waitMessage.message_id, escapeMarkdownV2Text(text), "Markdown");
},
onStop: async () => {
}
});
try {
src/commands/ollama-chat.ts
+33 -4
View File
@@ -6,12 +6,16 @@ import {
editMessageText,
escapeMarkdownV2Text,
extractText,
getPhotoMaxSize,
logError,
replyToMessage,
startIntervalEditor
} from "../util/utils";
import {Environment} from "../common/environment";
import {MessageStore} from "../common/message-store";
import axios from "axios";
import * as fs from "node:fs";
import path from "node:path";
export class OllamaChat extends ChatCommand {
regexp = /^\/ollama\s([^]+)/;
@@ -28,17 +32,40 @@ export class OllamaChat extends ChatCommand {
const chatId = msg.chat.id;
let imageFilePath: string | null = null;
const maxSize = await getPhotoMaxSize(msg.photo, 600);
if (maxSize) {
const res = await axios.get<ArrayBuffer>(maxSize.url, {responseType: "arraybuffer"});
const src = Buffer.from(res.data);
const imagePath = path.join(Environment.DATA_PATH, "temp");
if (!fs.existsSync(imagePath)) {
fs.mkdirSync(imagePath);
}
imageFilePath = path.join(imagePath, maxSize.unique_file_id + ".jpg");
try {
fs.writeFileSync(imageFilePath, src);
} catch (e) {
console.error(e);
imageFilePath = null;
}
}
const messageParts = await collectReplyChainText(msg);
console.log("MESSAGE PARTS", messageParts);
const chatMessages = messageParts.map(part => {
const chatMessages = messageParts.map((part, i) => {
return {
role: part.bot ? "ASSISTANT" : "USER",
content: extractText(part.content, Environment.BOT_PREFIX)
content: extractText(part.content, Environment.BOT_PREFIX),
images: imageFilePath && i === 0 ? [imageFilePath] : null
};
});
chatMessages.reverse();
chatMessages.unshift({role: "SYSTEM", content: Environment.SYSTEM_PROMPT});
chatMessages.unshift({role: "SYSTEM", content: Environment.SYSTEM_PROMPT, images: null});
let waitMessage: Message;
@@ -71,6 +98,8 @@ export class OllamaChat extends ChatCommand {
editFn: async (text) => {
await editMessageText(chatId, waitMessage.message_id, escapeMarkdownV2Text(text), "Markdown");
},
onStop: async () => {
}
});
try {
@@ -103,7 +132,7 @@ export class OllamaChat extends ChatCommand {
waitMessage.reply_to_message = msg;
waitMessage.text = currentText;
MessageStore.put(waitMessage);
await MessageStore.put(waitMessage);
await replyToMessage(waitMessage, `⏱️ ${diff}s`);
break;
src/commands/quote.ts
+4 -4
View File
@@ -298,16 +298,16 @@ async function getBackground(
const msgPhoto = photoArr && photoArr.length ? photoArr[photoArr.length - 1] : undefined;
if (msgPhoto?.file_id) {
const url = await getFileUrl(bot, msgPhoto.file_id);
const url = await getFileUrl(msgPhoto.file_id);
const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
src = Buffer.from(res.data);
} else {
if (author.userId) {
src = await getUserAvatar(bot, author.userId);
src = await getUserAvatar(author.userId);
} else if (author.chatId) {
src = await getChatAvatar(bot, author.chatId);
src = await getChatAvatar(author.chatId);
} else if (!isForwarded && reply.from?.id) {
src = await getUserAvatar(bot, reply.from.id);
src = await getUserAvatar(reply.from.id);
}
}
src/commands/system-specs.ts
+5 -5
View File
@@ -1,14 +1,14 @@
import {ChatCommand} from "../base/chat-command";
import {logError, oldSendMessage} from "../util/utils";
import {Message} from "typescript-telegram-bot-api";
import {systemSpecsText} from "../index";
import {systemInfoText} from "../index";
export class SystemSpecs implements ChatCommand {
regexp = /^\/systemspecs/i;
title = "/systemSpecs";
description = "System specifications of system";
regexp = /^\/systeminfo/i;
title = "/systemInfo";
description = "System information";
async execute(msg: Message) {
await oldSendMessage(msg, systemSpecsText).catch(logError);
await oldSendMessage(msg, systemInfoText).catch(logError);
}
}
src/db/message-dao.ts
+3 -2
View File
@@ -5,7 +5,8 @@ import {and, eq} from "drizzle-orm";
import {inArray} from "drizzle-orm/sql/expressions/conditions";
import {Message} from "typescript-telegram-bot-api";
import {Dao} from "../base/dao";
import {buildExcludedSet} from "../util/utils";
import {buildExcludedSet, extractTextMessage} from "../util/utils";
import {Environment} from "../common/environment";
export class MessageDao extends Dao<StoredMessage> {
@@ -88,7 +89,7 @@ export class MessageDao extends Dao<StoredMessage> {
id: msg.message_id,
replyToMessageId: msg.reply_to_message?.message_id,
fromId: msg.from.id,
text: msg.text,
text: extractTextMessage(msg, Environment.BOT_PREFIX),
date: msg.date,
firstName: msg.from.first_name,
lastName: msg.from.last_name,
src/index.ts
+17 -21
View File
@@ -72,10 +72,10 @@ export const ollama = new Ollama({
export const googleAi = new GoogleGenAI({apiKey: Environment.GEMINI_API_KEY});
export let systemSpecsText: string = "";
export let systemInfoText: string = "";
export function setSystemSpecs(systemSpecs: string) {
systemSpecsText = systemSpecs;
export function setSystemInfo(info: string) {
systemInfoText = info;
}
export const chatCommands: ChatCommand[] = [
@@ -149,22 +149,6 @@ bot.on("my_chat_member", async (u) => {
console.log("my_chat_member", u);
});
bot.on("message", async (message) => {
console.log("message", message);
await UserStore.put(message.from);
if ((message.new_chat_members?.length || 0 > 0)) {
await bot.sendMessage({chat_id: message.chat.id, text: randomValue(inviteAnswers)}).catch(logError);
return;
}
if (message.left_chat_member && message.left_chat_member.id !== botUser.id) {
await bot.sendMessage({chat_id: message.chat.id, text: randomValue(kickAnswers)}).catch(logError);
return;
}
});
bot.on("edited_message", async (msg) => {
console.log("edited_message", msg);
@@ -175,8 +159,20 @@ bot.on("edited_message", async (msg) => {
await MessageStore.put(msg);
});
bot.on("message:text", async (msg) => {
await MessageStore.put(msg);
bot.on("message", async (msg) => {
console.log("message", msg);
await Promise.all([MessageStore.put(msg), UserStore.put(msg.from)]);
if ((msg.new_chat_members?.length || 0 > 0)) {
await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(inviteAnswers)}).catch(logError);
return;
}
if (msg.left_chat_member && msg.left_chat_member.id !== botUser.id) {
await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(kickAnswers)}).catch(logError);
return;
}
if (muted.has(msg.from.id)) return;
src/util/utils.ts
+45 -18
View File
@@ -1,18 +1,10 @@
import * as si from "systeminformation";
import {ChatCommand} from "../base/chat-command";
import {CallbackCommand} from "../base/callback-command";
import {
CallbackQuery,
InlineKeyboardMarkup,
Message,
ParseMode,
PhotoSize,
TelegramBot,
User
} from "typescript-telegram-bot-api";
import {CallbackQuery, InlineKeyboardMarkup, Message, ParseMode, PhotoSize, User} from "typescript-telegram-bot-api";
import {Environment} from "../common/environment";
import {TelegramError} from "typescript-telegram-bot-api/dist/errors";
import {bot, botUser, messageDao, setSystemSpecs} from "../index";
import {bot, botUser, messageDao, setSystemInfo} from "../index";
import os from "os";
import axios from "axios";
import {MessagePart} from "../common/message-part";
@@ -240,7 +232,7 @@ export async function initSystemSpecs(): Promise<void> {
`CPU: ${cpu.manufacturer} ${cpu.brand} ${cpu.physicalCores} cores ${cpu.cores} threads\n` +
`RAM: ${ramSize} GB`;
setSystemSpecs(text);
setSystemInfo(text);
return Promise.resolve();
} catch (e) {
return Promise.reject(e);
@@ -335,18 +327,18 @@ export function escapeMarkdownV2Text(s: string) {
return s;
}
export async function getFileUrl(bot: TelegramBot, fileId: string) {
export async function getFileUrl(fileId: string): Promise<string> {
const file = await bot.getFile({file_id: fileId});
return `https://api.telegram.org/file/bot${bot.botToken}/${file.file_path}`;
}
export async function getChatAvatar(bot: TelegramBot, chatId: number): Promise<Buffer | null> {
export async function getChatAvatar(chatId: number): Promise<Buffer | null> {
try {
const chat = await bot.getChat({chat_id: chatId});
const photo = chat?.photo?.big_file_id || chat?.photo?.small_file_id;
if (!photo) return null;
const url = await getFileUrl(bot, photo);
const url = await getFileUrl(photo);
const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
return Buffer.from(res.data);
} catch {
@@ -354,12 +346,12 @@ export async function getChatAvatar(bot: TelegramBot, chatId: number): Promise<B
}
}
export async function getUserAvatar(bot: TelegramBot, userId: number): Promise<Buffer | null> {
export async function getUserAvatar(userId: number): Promise<Buffer | null> {
const photos = await bot.getUserProfilePhotos({user_id: userId, limit: 1});
const last: PhotoSize | undefined = photos.photos?.[0]?.[photos.photos[0].length - 1];
if (!last) return null;
const url = await getFileUrl(bot, last.file_id);
const url = await getFileUrl(last.file_id);
const res = await axios.get<ArrayBuffer>(url, {responseType: "arraybuffer"});
return Buffer.from(res.data);
}
@@ -401,7 +393,7 @@ export async function collectReplyChainText(triggerMsg: Message, prefix: string
const t = extractTextMessage(triggerMsg, prefix);
if (t) parts.push({
bot: triggerMsg.from.id === botUser.id,
content: triggerMsg.text,
content: t,
name: triggerMsg.from.first_name
});
}
@@ -499,7 +491,7 @@ export async function waveDistortSharp(
if (sx < 0 || sx >= width || sy < 0 || sy >= height) {
// прозрачный пиксель
out[di + 0] = 0;
out[di] = 0;
out[di + 1] = 0;
out[di + 2] = 0;
out[di + 3] = 0;
@@ -678,6 +670,7 @@ export function startIntervalEditor(params: {
intervalMs: number;
getText: () => string;
editFn: (text: string) => Promise<void>;
onStop: () => Promise<void>;
}) {
let lastSent = "";
let stopped = false;
@@ -706,6 +699,7 @@ export function startIntervalEditor(params: {
stopped = true;
clearInterval(timer);
await tick();
await params.onStop();
},
};
}
@@ -757,3 +751,36 @@ export function getRuntimeInfo(): RuntimeInfo {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {runtime: "unknown", version: String((process as any).version ?? "")};
}
export type PhotoMaxSize = { width: number, height: number, url: string; unique_file_id: string; };
export async function getPhotoMaxSize(photos: PhotoSize[], target: number = 1280): Promise<PhotoMaxSize | null> {
if (!photos) return null;
photos = photos.filter(p => Math.max(p.width, p.height) <= target);
if (photos.length === 0) return null;
if (photos.length === 1) {
return mapPhotoSizeToMax(photos[0]);
}
const max = photos.reduce((prev, cur) => {
if (!prev) return cur;
return cur.width * cur.height > prev.width * prev.height ? cur : prev;
}, null);
if (!max) return null;
return mapPhotoSizeToMax(max);
}
export async function mapPhotoSizeToMax(size: PhotoSize): Promise<PhotoMaxSize | null> {
if (!size) return null;
return {
width: size.width,
height: size.height,
url: await getFileUrl(size.file_id),
unique_file_id: size.file_unique_id
};
}