fix(images): cache Telegram photos by unique_id and pass base64 to LLM commands

Stop reading image files inside gemini/mistral commands; use pre-encoded image data from message parts

Rework loadImagesIfExists / loadImagesFromFileIds to return cached file_unique_ids and download only missing sizes

Encode cached images to base64 when assembling the reply chain

Make getPhotoMaxSize synchronous (returns PhotoSize), and map to URL only when needed

Await MessageStore.put / UserStore.put and prefetch single-image downloads on message receipt

src/commands/gemini-chat.ts

@@ -13,7 +13,6 @@ import {
     oldReplyToMessage,
     startIntervalEditor
 } from "../util/utils";
-import fs from "node:fs";
 
 export class GeminiChat extends ChatCommand {
     command = "gemini";
@@ -66,10 +65,9 @@ export class GeminiChat extends ChatCommand {
             const images = messageParts[0].images;
 
             images.forEach(image => {
-                const base64Image = Buffer.from(fs.readFileSync(image)).toString("base64");
                 input.push({
                     type: "image",
-                    data: base64Image,
+                    data: image,
                     mime_type: "image/png"
                 });
             });

src/commands/mistral-chat.ts

@@ -7,13 +7,11 @@ import {
     escapeMarkdownV2Text,
     logError,
     oldReplyToMessage,
-    photoPathByUniqueId,
     startIntervalEditor
 } from "../util/utils";
 import {Environment} from "../common/environment";
 import {bot, mistralAi} from "../index";
 import {MessageStore} from "../common/message-store";
-import fs from "node:fs";
 
 export class MistralChat extends ChatCommand {
     command = "mistral";
@@ -46,10 +44,9 @@ export class MistralChat extends ChatCommand {
             });
 
             for (const image of part.images) {
-                const base64Image = Buffer.from(fs.readFileSync(photoPathByUniqueId(image))).toString("base64");
                 content.push({
                     type: "image_url",
-                    imageUrl: "data:image/jpeg;base64," + base64Image
+                    imageUrl: "data:image/jpeg;base64," + image
                 });
             }
 

src/common/message-store.ts

@@ -1,6 +1,6 @@
 import {StoredMessage} from "../model/stored-message";
 import {Message} from "typescript-telegram-bot-api";
-import {extractTextMessage, isStoredMessage} from "../util/utils";
+import {extractTextMessage, getPhotoMaxSize, isStoredMessage} from "../util/utils";
 import {messageDao} from "../index";
 
 export class MessageStore {
@@ -22,6 +22,7 @@ export class MessageStore {
             fromId: m.from.id,
             text: extractTextMessage(m),
             date: m.date ?? 0,
+            photoMaxSizeFilePath: m.photo ? [getPhotoMaxSize(m.photo).file_unique_id] : null
         };
 
         this.map.set(this.key(msg.chatId, msg.id), msg);

src/common/user-store.ts

@@ -9,7 +9,7 @@ export class UserStore {
         return this.map;
     }
 
-    static async put(u: User) {
+    static async put(u: User): Promise<StoredUser> {
         const user: StoredUser = {
             id: u.id,
             isBot: u.is_bot,
@@ -22,6 +22,7 @@ export class UserStore {
         this.map.set(u.id, user);
 
         await userDao.insert(userDao.mapTo([u]));
+        return user;
     }
 
     static async get(id: number): Promise<StoredUser | null> {

src/util/utils.ts

@@ -497,9 +497,18 @@ export async function loadImagesIfExists(msg: Message | StoredMessage): Promise<
         return msg.photoMaxSizeFilePath;
     }
 
+    if (!msg.photo?.length) return;
+
     const imageFilePaths: string[] = [];
 
-    const maxSize = await getPhotoMaxSize(msg.photo);
+    for (const size of msg.photo) {
+        const exists = fs.existsSync(photoPathByUniqueId(size.file_unique_id));
+        if (exists) {
+            return [size.file_unique_id];
+        }
+    }
+
+    const maxSize = await mapPhotoSizeToMax(getPhotoMaxSize(msg.photo));
     if (maxSize) {
         const imagePath = path.join(Environment.DATA_PATH, "temp");
         if (!fs.existsSync(imagePath)) {
@@ -527,14 +536,23 @@ export async function loadImagesIfExists(msg: Message | StoredMessage): Promise<
     return imageFilePaths;
 }
 
-export async function loadImagesFromFileIds(maxSizes: PhotoMaxSize[]): Promise<string[] | null> {
+export async function loadImagesFromFileIds(sizes: PhotoSize[]): Promise<string[] | null> {
-    if (!maxSizes?.length) return null;
+    if (!sizes?.length) return null;
 
     const dataPath = path.join(Environment.DATA_PATH, "temp");
     if (!fs.existsSync(dataPath)) {
         fs.mkdirSync(dataPath);
     }
 
+    const existing =
+        sizes.filter(s => fs.existsSync(photoPathByUniqueId(s.file_unique_id)))
+            .map(s => s.file_unique_id);
+
+    const promises = sizes.filter(s => !fs.existsSync(photoPathByUniqueId(s.file_unique_id)))
+        .map(s => mapPhotoSizeToMax(s));
+
+    const maxSizes = await Promise.all(promises);
+
     const imagePromises = maxSizes.map((size) => {
         return axios.get<ArrayBuffer>(size.url, {responseType: "arraybuffer"});
     });
@@ -542,17 +560,19 @@ export async function loadImagesFromFileIds(maxSizes: PhotoMaxSize[]): Promise<s
     const responses = await Promise.all(imagePromises);
     const paths = responses.map((res, index) => {
         try {
-            const imageFilePath = path.join(dataPath, maxSizes[index].unique_file_id + ".jpg");
+            const uniqueFileId = maxSizes[index].unique_file_id;
+            const imageFilePath = path.join(dataPath, uniqueFileId + ".jpg");
             const src = Buffer.from(res.data);
             fs.writeFileSync(imageFilePath, src);
-            return imageFilePath;
+            return uniqueFileId;
         } catch (e) {
             logError(e);
             return null;
         }
     });
-
+    const finalPaths = paths.filter(p => p);
-    return paths.filter(p => p);
+    finalPaths.unshift(...existing);
+    return finalPaths;
 }
 
 export async function collectReplyChainText(triggerMsg: Message | StoredMessage, limit: number = 40, includeTrigger = true, cutPrefix: boolean = true): Promise<MessagePart[]> {
@@ -561,15 +581,20 @@ export async function collectReplyChainText(triggerMsg: Message | StoredMessage,
     const pushPart = async (msg: Message | StoredMessage, textRequired: boolean = false) => {
         const rawText = extractTextMessage(msg);
         const cleanText = cutPrefix ? cutPrefixes(rawText) : rawText;
-        const images = await loadImagesIfExists(msg);
+        const imageNames = await loadImagesIfExists(msg);
 
         if (!cleanText && textRequired) return;
-        if (!cleanText && !images?.length) return;
+        if (!cleanText && !imageNames?.length) return;
 
         const fromId = isStoredMessage(msg) ? msg.fromId : msg.from.id;
         const firstName = isStoredMessage(msg) ?
             (await UserStore.get(msg.fromId))?.firstName : msg.from.first_name;
 
+        const images = imageNames ? imageNames.map(n => {
+            const filePath = photoPathByUniqueId(n);
+            return Buffer.from(fs.readFileSync(filePath)).toString("base64");
+        }) : null;
+
         parts.push({
             bot: fromId === botUser.id,
             content: cleanText ? cleanText : "",
@@ -932,7 +957,7 @@ export function getRuntimeInfo(): RuntimeInfo {
 
 export type PhotoMaxSize = { width: number, height: number, url: string; file_id: string; unique_file_id: string; };
 
-export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Environment.MAX_PHOTO_SIZE): Promise<PhotoMaxSize | null> {
+export function getPhotoMaxSize(photos: PhotoSize[], target: number = Environment.MAX_PHOTO_SIZE): PhotoSize | null {
     if (!photos) return null;
 
     photos = photos.filter(p => Math.max(p.width, p.height) <= target);
@@ -940,7 +965,7 @@ export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Envi
     if (photos.length === 0) return null;
 
     if (photos.length === 1) {
-        return mapPhotoSizeToMax(photos[0]);
+        return photos[0];
     }
 
     const max = photos.reduce((prev, cur) => {
@@ -949,8 +974,7 @@ export async function getPhotoMaxSize(photos: PhotoSize[], target: number = Envi
         return cur.width * cur.height > prev.width * prev.height ? cur : prev;
     }, null);
 
-    if (!max) return null;
+    return max;
-    return mapPhotoSizeToMax(max);
 }
 
 export async function mapPhotoSizeToMax(size: PhotoSize): Promise<PhotoMaxSize | null> {
@@ -993,14 +1017,21 @@ export async function processNewMessage(msg: Message) {
     console.log("message", msg);
 
     let storedMsg: StoredMessage | null = null;
-    Promise.all([
+
-            MessageStore.put(msg).then(r => {
+    try {
-                storedMsg = r;
+        const results = await Promise.all([
-                console.log("storedMsg", storedMsg);
+                MessageStore.put(msg),
-            }),
                 UserStore.put(msg.from)
             ]
-    ).catch(logError);
+        );
+
+        storedMsg = results[0];
+        if (!msg.media_group_id && storedMsg.photoMaxSizeFilePath) {
+            await loadImagesIfExists(msg);
+        }
+    } catch (e) {
+        logError(e);
+    }
 
     if ((msg.new_chat_members?.length || 0 > 0)) {
         await bot.sendMessage({chat_id: msg.chat.id, text: randomValue(Environment.ANSWERS.invite)}).catch(logError);
@@ -1080,9 +1111,7 @@ async function processAlbum(groupId: string): Promise<string[]> {
         .map(m => m.photo);
 
     const allPhotoMaxSizes = await Promise.all(allPhotos.map(photo => getPhotoMaxSize(photo)));
-    const ids = allPhotoMaxSizes.map(p => p.unique_file_id);
+    const ids = await loadImagesFromFileIds(allPhotoMaxSizes);
-
-    await loadImagesFromFileIds(allPhotoMaxSizes);
 
     console.log(`Received album ${groupId} with ${ids.length} photos.`);
     console.log("File IDs:", ids);