diff --git a/src/commands/gemini-chat.ts b/src/commands/gemini-chat.ts index 778adf9..b2c3282 100644 --- a/src/commands/gemini-chat.ts +++ b/src/commands/gemini-chat.ts @@ -13,7 +13,6 @@ import { oldReplyToMessage, startIntervalEditor } from "../util/utils"; -import fs from "node:fs"; export class GeminiChat extends ChatCommand { command = "gemini"; @@ -66,10 +65,9 @@ export class GeminiChat extends ChatCommand { const images = messageParts[0].images; images.forEach(image => { - const base64Image = Buffer.from(fs.readFileSync(image)).toString("base64"); input.push({ type: "image", - data: base64Image, + data: image, mime_type: "image/png" }); }); diff --git a/src/commands/mistral-chat.ts b/src/commands/mistral-chat.ts index 5d566e4..6566ba7 100644 --- a/src/commands/mistral-chat.ts +++ b/src/commands/mistral-chat.ts @@ -7,13 +7,11 @@ import { escapeMarkdownV2Text, logError, oldReplyToMessage, - photoPathByUniqueId, startIntervalEditor } from "../util/utils"; import {Environment} from "../common/environment"; import {bot, mistralAi} from "../index"; import {MessageStore} from "../common/message-store"; -import fs from "node:fs"; export class MistralChat extends ChatCommand { command = "mistral"; @@ -46,10 +44,9 @@ export class MistralChat extends ChatCommand { }); for (const image of part.images) { - const base64Image = Buffer.from(fs.readFileSync(photoPathByUniqueId(image))).toString("base64"); content.push({ type: "image_url", - imageUrl: "data:image/jpeg;base64," + base64Image + imageUrl: "data:image/jpeg;base64," + image }); } diff --git a/src/common/message-store.ts b/src/common/message-store.ts index cfed829..f067906 100644 --- a/src/common/message-store.ts +++ b/src/common/message-store.ts @@ -1,6 +1,6 @@ import {StoredMessage} from "../model/stored-message"; import {Message} from "typescript-telegram-bot-api"; -import {extractTextMessage, isStoredMessage} from "../util/utils"; +import {extractTextMessage, getPhotoMaxSize, isStoredMessage} from "../util/utils"; import {messageDao} from "../index"; export class MessageStore { @@ -22,6 +22,7 @@ export class MessageStore { fromId: m.from.id, text: extractTextMessage(m), date: m.date ?? 0, + photoMaxSizeFilePath: m.photo ? [getPhotoMaxSize(m.photo).file_unique_id] : null }; this.map.set(this.key(msg.chatId, msg.id), msg); diff --git a/src/common/user-store.ts b/src/common/user-store.ts index d836986..1689252 100644 --- a/src/common/user-store.ts +++ b/src/common/user-store.ts @@ -9,7 +9,7 @@ export class UserStore { return this.map; } - static async put(u: User) { + static async put(u: User): Promise { const user: StoredUser = { id: u.id, isBot: u.is_bot, @@ -22,6 +22,7 @@ export class UserStore { this.map.set(u.id, user); await userDao.insert(userDao.mapTo([u])); + return user; } static async get(id: number): Promise { diff --git a/src/util/utils.ts b/src/util/utils.ts index 6d617a0..bd80731 100644 --- a/src/util/utils.ts +++ b/src/util/utils.ts @@ -497,9 +497,18 @@ export async function loadImagesIfExists(msg: Message | StoredMessage): Promise< return msg.photoMaxSizeFilePath; } + if (!msg.photo?.length) return; + const imageFilePaths: string[] = []; - const maxSize = await getPhotoMaxSize(msg.photo); + for (const size of msg.photo) { + const exists = fs.existsSync(photoPathByUniqueId(size.file_unique_id)); + if (exists) { + return [size.file_unique_id]; + } + } + + const maxSize = await mapPhotoSizeToMax(getPhotoMaxSize(msg.photo)); if (maxSize) { const imagePath = path.join(Environment.DATA_PATH, "temp"); if (!fs.existsSync(imagePath)) { @@ -527,14 +536,23 @@ export async function loadImagesIfExists(msg: Message | StoredMessage): Promise< return imageFilePaths; } -export async function loadImagesFromFileIds(maxSizes: PhotoMaxSize[]): Promise { - if (!maxSizes?.length) return null; +export async function loadImagesFromFileIds(sizes: PhotoSize[]): Promise { + if (!sizes?.length) return null; const dataPath = path.join(Environment.DATA_PATH, "temp"); if (!fs.existsSync(dataPath)) { fs.mkdirSync(dataPath); } + const existing = + sizes.filter(s => fs.existsSync(photoPathByUniqueId(s.file_unique_id))) + .map(s => s.file_unique_id); + + const promises = sizes.filter(s => !fs.existsSync(photoPathByUniqueId(s.file_unique_id))) + .map(s => mapPhotoSizeToMax(s)); + + const maxSizes = await Promise.all(promises); + const imagePromises = maxSizes.map((size) => { return axios.get(size.url, {responseType: "arraybuffer"}); }); @@ -542,17 +560,19 @@ export async function loadImagesFromFileIds(maxSizes: PhotoMaxSize[]): Promise { try { - const imageFilePath = path.join(dataPath, maxSizes[index].unique_file_id + ".jpg"); + const uniqueFileId = maxSizes[index].unique_file_id; + const imageFilePath = path.join(dataPath, uniqueFileId + ".jpg"); const src = Buffer.from(res.data); fs.writeFileSync(imageFilePath, src); - return imageFilePath; + return uniqueFileId; } catch (e) { logError(e); return null; } }); - - return paths.filter(p => p); + const finalPaths = paths.filter(p => p); + finalPaths.unshift(...existing); + return finalPaths; } export async function collectReplyChainText(triggerMsg: Message | StoredMessage, limit: number = 40, includeTrigger = true, cutPrefix: boolean = true): Promise { @@ -561,15 +581,20 @@ export async function collectReplyChainText(triggerMsg: Message | StoredMessage, const pushPart = async (msg: Message | StoredMessage, textRequired: boolean = false) => { const rawText = extractTextMessage(msg); const cleanText = cutPrefix ? cutPrefixes(rawText) : rawText; - const images = await loadImagesIfExists(msg); + const imageNames = await loadImagesIfExists(msg); if (!cleanText && textRequired) return; - if (!cleanText && !images?.length) return; + if (!cleanText && !imageNames?.length) return; const fromId = isStoredMessage(msg) ? msg.fromId : msg.from.id; const firstName = isStoredMessage(msg) ? (await UserStore.get(msg.fromId))?.firstName : msg.from.first_name; + const images = imageNames ? imageNames.map(n => { + const filePath = photoPathByUniqueId(n); + return Buffer.from(fs.readFileSync(filePath)).toString("base64"); + }) : null; + parts.push({ bot: fromId === botUser.id, content: cleanText ? cleanText : "", @@ -932,7 +957,7 @@ export function getRuntimeInfo(): RuntimeInfo { export type PhotoMaxSize = { width: number, height: number, url: string; file_id: string; unique_file_id: string; }; -export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Environment.MAX_PHOTO_SIZE): Promise { +export function getPhotoMaxSize(photos: PhotoSize[], target: number = Environment.MAX_PHOTO_SIZE): PhotoSize | null { if (!photos) return null; photos = photos.filter(p => Math.max(p.width, p.height) <= target); @@ -940,7 +965,7 @@ export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Envi if (photos.length === 0) return null; if (photos.length === 1) { - return mapPhotoSizeToMax(photos[0]); + return photos[0]; } const max = photos.reduce((prev, cur) => { @@ -949,8 +974,7 @@ export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Envi return cur.width * cur.height > prev.width * prev.height ? cur : prev; }, null); - if (!max) return null; - return mapPhotoSizeToMax(max); + return max; } export async function mapPhotoSizeToMax(size: PhotoSize): Promise { @@ -993,14 +1017,21 @@ export async function processNewMessage(msg: Message) { console.log("message", msg); let storedMsg: StoredMessage | null = null; - Promise.all([ - MessageStore.put(msg).then(r => { - storedMsg = r; - console.log("storedMsg", storedMsg); - }), - UserStore.put(msg.from) - ] - ).catch(logError); + + try { + const results = await Promise.all([ + MessageStore.put(msg), + UserStore.put(msg.from) + ] + ); + + storedMsg = results[0]; + if (!msg.media_group_id && storedMsg.photoMaxSizeFilePath) { + await loadImagesIfExists(msg); + } + } catch (e) { + logError(e); + } if ((msg.new_chat_members?.length || 0 > 0)) { await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(Environment.ANSWERS.invite)}).catch(logError); @@ -1080,9 +1111,7 @@ async function processAlbum(groupId: string): Promise { .map(m => m.photo); const allPhotoMaxSizes = await Promise.all(allPhotos.map(photo => getPhotoMaxSize(photo))); - const ids = allPhotoMaxSizes.map(p => p.unique_file_id); - - await loadImagesFromFileIds(allPhotoMaxSizes); + const ids = await loadImagesFromFileIds(allPhotoMaxSizes); console.log(`Received album ${groupId} with ${ids.length} photos.`); console.log("File IDs:", ids);