shitton of the ai changes

2026-05-01 04:54:11 +03:00
parent d95c37a322
commit 8cff086a8e
194 changed files with 29409 additions and 8841 deletions
@@ -3,6 +3,7 @@ import {Environment} from "../common/environment";
 import {logError} from "../util/utils";
 import {Answers} from "../model/answers";
 import path from "node:path";
+import {KeyedAsyncLock} from "../util/async-lock";

 type DataJsonFile = {
    admins: number[]
@@ -11,9 +12,42 @@ type DataJsonFile = {

 export let jsonFile: DataJsonFile;

+const DEFAULT_DATA: DataJsonFile = {
+    admins: [],
+    muted: [],
+};
+
+const DEFAULT_ANSWERS: Answers = {
+    test: ["a"],
+    prefix: ["?"],
+    better: ["Better"],
+    who: [],
+    kick: [],
+    invite: [],
+    day: [],
+};
+
+const dataFileLock = new KeyedAsyncLock();
+
+function ensureDataPath(): void {
+    fs.mkdirSync(Environment.DATA_PATH, {recursive: true});
+}
+
+function readJsonFile<T>(fileName: string, defaultValue: T): T {
+    ensureDataPath();
+
+    const filePath = `${Environment.DATA_PATH}/${fileName}`;
+    if (!fs.existsSync(filePath)) {
+        fs.writeFileSync(filePath, JSON.stringify(defaultValue, null, 2));
+        return structuredClone(defaultValue);
+    }
+
+    return JSON.parse(fs.readFileSync(filePath).toString()) as T;
+}
+
 export async function readData(): Promise<void> {
    try {
-        jsonFile = JSON.parse(fs.readFileSync(`${Environment.DATA_PATH}/data.json`).toString());
+        jsonFile = readJsonFile("data.json", DEFAULT_DATA);

        const admins = jsonFile.admins || [];
        admins.unshift(Environment.CREATOR_ID);
@@ -23,48 +57,43 @@ export async function readData(): Promise<void> {

        return Promise.resolve();
    } catch (e) {
-        logError(e);
+        logError(e instanceof Error ? e : String(e));
        return Promise.reject(e);
    }
 }

-export async function readPrompts(): Promise<void> {
-    try {
-        const prompt = fs.readFileSync(path.join(Environment.DATA_PATH, "system_prompt.txt")).toString().trim();
-        if (prompt.length) {
-            Environment.setSystemPrompt(prompt);
-        }
-    } catch (e) {
-        logError(e);
-    }
-
-    return Promise.resolve();
-}
-
 export async function saveData(): Promise<void> {
-    const adminIds: number[] = [];
-    Environment.ADMIN_IDS.forEach(id => adminIds.push(id));
-    jsonFile.admins = adminIds;
+    return dataFileLock.runExclusive("data.json", async () => {
+        ensureDataPath();
+        jsonFile ??= structuredClone(DEFAULT_DATA);

-    const mutedList: number[] = [];
-    Environment.MUTED_IDS.forEach(id => mutedList.push(id));
-    jsonFile.muted = mutedList;
+        const adminIds: number[] = [];
+        Environment.ADMIN_IDS.forEach(id => adminIds.push(id));
+        jsonFile.admins = adminIds;

-    try {
-        fs.writeFileSync(`${Environment.DATA_PATH}/data.json`, JSON.stringify(jsonFile));
-        return readData();
-    } catch (e) {
-        return Promise.reject(e);
-    }
+        const mutedList: number[] = [];
+        Environment.MUTED_IDS.forEach(id => mutedList.push(id));
+        jsonFile.muted = mutedList;
+
+        try {
+            const filePath = path.join(Environment.DATA_PATH, "data.json");
+            const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+            fs.writeFileSync(tmpPath, JSON.stringify(jsonFile));
+            fs.renameSync(tmpPath, filePath);
+            return readData();
+        } catch (e) {
+            return Promise.reject(e);
+        }
+    });
 }

 export async function retrieveAnswers(): Promise<void> {
    try {
-        const json: Answers = JSON.parse(fs.readFileSync(`${Environment.DATA_PATH}/answers.json`).toString());
+        const json = readJsonFile("answers.json", DEFAULT_ANSWERS);
        Environment.setAnswers(json);
        return Promise.resolve();
    } catch (e) {
-        logError(e);
+        logError(e instanceof Error ? e : String(e));
        return Promise.reject(e);
    }
-}
+}
@@ -0,0 +1,89 @@
+export type UserDbRow = {
+    id: number;
+    isBot: number;
+    firstName: string;
+    lastName: string | null;
+    userName: string | null;
+    isPremium: number | null;
+    langCode: string | null;
+    interfaceLanguage: string | null;
+    aiProvider: string | null;
+    aiResponseLanguage: string | null;
+    aiContextSize: number | null;
+    aiVoiceMode: string | null;
+    aiImageOutputMode: string | null;
+};
+
+export type MessageDbRow = {
+    id: number;
+    chatId: number;
+    replyToMessageId: number | null;
+    fromId: number;
+    text: string | null;
+    quoteText: string | null;
+    date: number;
+    deletedByBotAt: number | null;
+    attachments: string | null;
+    pipelineAudit: string | null;
+};
+
+export type AttachmentDbRow = {
+    id: string;
+    messageChatId: number;
+    messageId: number;
+    direction: string;
+    scope: string;
+    kind: string;
+    artifactKind: string | null;
+    fileId: string;
+    fileUniqueId: string | null;
+    fileName: string;
+    mimeType: string | null;
+    cachePath: string;
+    sizeBytes: number | null;
+    sha256: string | null;
+    metadata: string | null;
+    createdAt: string;
+};
+
+export type ArtifactDbRow = {
+    id: string;
+    requestId: string;
+    messageChatId: number;
+    messageId: number;
+    kind: string;
+    stage: string;
+    attachmentId: string | null;
+    payload: string;
+    createdAt: string;
+};
+
+export type RequestAuditDbRow = {
+    id: string;
+    requestId: string;
+    messageChatId: number;
+    messageId: number;
+    stage: string;
+    status: string;
+    startedAt: string | null;
+    finishedAt: string | null;
+    durationMs: number | null;
+    provider: string | null;
+    model: string | null;
+    details: string | null;
+    error: string | null;
+};
+
+export type AiRequestDbRow = {
+    requestId: string;
+    chatId: number;
+    messageId: number;
+    responseMessageId: number | null;
+    fromId: number;
+    provider: string;
+    model: string;
+    status: string;
+    startedAt: string;
+    finishedAt: string | null;
+    error: string | null;
+};
@@ -1,110 +1,182 @@
-import {MessageInsert, messagesTable} from "./schema";
 import {DatabaseManager} from "./database-manager";
 import {StoredMessage} from "../model/stored-message";
-import {and, eq} from "drizzle-orm";
-import {inArray} from "drizzle-orm/sql/expressions/conditions";
 import {Dao} from "../base/dao";
-import {buildExcludedSet} from "../util/utils";
+import {appLogger} from "../logging/logger";
+import {StoredAttachment} from "../model/stored-attachment";
+import {MessageDbRow} from "./db-types";
+import type {PipelineAuditEvent} from "../ai/user-request-pipeline";

-export class MessageDao extends Dao<StoredMessage> {
+export class MessageDao extends Dao<StoredMessage, {chatId: number; id: number}, {chatId: number; ids: number[]}, MessageDbRow[]> {

-    private tag: string = "MessageDao";
+    private readonly logger = appLogger.child("dao:messages");

    override async getAll(): Promise<StoredMessage[]> {
        const then = Date.now();

-        const messages = await DatabaseManager.db.select().from(messagesTable);
+        const messages = await DatabaseManager.getAllMessages();
+        const hydrated = await this.hydrateMissingMessageData(messages);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getAll()`, `took ${diff}ms; size: ${messages.length}`);
+        this.logger.trace("get_all", {dao: "messages", duration: `${diff}ms`, size: hydrated.length});

-        return this.mapFrom(messages);
+        return this.mapFrom(hydrated);
    }

    override async getById(params: { chatId: number, id: number }): Promise<StoredMessage | null> {
        const then = Date.now();

-        const messages =
-            await DatabaseManager.db.select()
-                .from(messagesTable)
-                .where(
-                    and(
-                        eq(messagesTable.chatId, params.chatId),
-                        eq(messagesTable.id, params.id)
-                    )
-                );
+        const message = await DatabaseManager.getMessageById(params.chatId, params.id);
+        const hydrated = await this.hydrateMissingMessageData(message ? [message] : []);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getById(${params.chatId}, ${params.id})`, `took ${diff}ms; size: ${messages.length}`);
+        this.logger.trace("get_by_id", {dao: "messages", chatId: params.chatId, id: params.id, duration: `${diff}ms`, size: hydrated.length});

-        const m = messages[0];
-        if (!m) return null;
-        return this.mapFrom([m])[0];
+        if (!hydrated.length) return null;
+        return this.mapFrom(hydrated)[0];
    }

    override async getByIds(params: { chatId: number, ids: number[] }): Promise<StoredMessage[]> {
        const then = Date.now();

-        const messages =
-            await DatabaseManager.db.select()
-                .from(messagesTable)
-                .where(
-                    and(
-                        eq(messagesTable.chatId, params.chatId),
-                        inArray(messagesTable.id, params.ids)
-                    )
-                );
+        const messages = await DatabaseManager.getMessagesByIds(params.chatId, params.ids);
+        const hydrated = await this.hydrateMissingMessageData(messages);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getByIds(${params.chatId}, ${params.ids})`, `took ${diff}ms; size: ${messages.length}`);
+        this.logger.trace("get_by_ids", {dao: "messages", chatId: params.chatId, ids: params.ids, duration: `${diff}ms`, size: hydrated.length});

-        return this.mapFrom(messages);
+        return this.mapFrom(hydrated);
    }

-    async insert(values: MessageInsert[]): Promise<true> {
+    async insert(values: MessageDbRow[]): Promise<true> {
+        if (!values.length) return true;
+
        const then = Date.now();
-        const r = await DatabaseManager.db
-            .insert(messagesTable)
-            .values(values)
-            .onConflictDoUpdate({
-                target: messagesTable.id,
-                set: buildExcludedSet(messagesTable, ["id"])
-            });
+        await DatabaseManager.upsertMessages(values);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: insert(size: ${values.length})`, `took ${diff}ms'; inserted: ${r.rowsAffected}`);
+        this.logger.debug("insert", {dao: "messages", duration: `${diff}ms`, size: values.length});
        return true;
    }

-    mapStoredTo(messages: StoredMessage[]): MessageInsert[] {
+    mapStoredTo(messages: StoredMessage[]): MessageDbRow[] {
        return messages.map(msg => {
            return {
                chatId: msg.chatId,
                id: msg.id,
-                replyToMessageId: msg.replyToMessageId,
+                replyToMessageId: msg.replyToMessageId ?? null,
                fromId: msg.fromId,
-                text: msg.text,
+                text: msg.text ?? null,
+                quoteText: msg.quoteText ?? null,
                date: msg.date,
-                photoMaxSizeFilePath: msg.photoMaxSizeFilePath?.join(";"),
+                deletedByBotAt: msg.deletedByBotAt ?? null,
+                attachments: msg.attachments?.length ? JSON.stringify(msg.attachments) : null,
+                pipelineAudit: msg.pipelineAudit?.length ? JSON.stringify(msg.pipelineAudit) : null,
            };
        });
    }

-    mapFrom(messages: MessageInsert[]): StoredMessage[] {
+    mapFrom(messages: MessageDbRow[]): StoredMessage[] {
        return messages.map(m => {
            return {
                chatId: m.chatId,
                id: m.id,
-                replyToMessageId: m.replyToMessageId,
+                replyToMessageId: m.replyToMessageId || undefined,
                fromId: m.fromId,
                text: m.text,
+                quoteText: m.quoteText,
                date: m.date,
-                photoMaxSizeFilePath: m.photoMaxSizeFilePath?.split(";")
+                deletedByBotAt: m.deletedByBotAt,
+                attachments: parseAttachments(m.attachments),
+                pipelineAudit: parsePipelineAudit(m.pipelineAudit),
            };
        });
    }
-}
+
+    private async hydrateMissingMessageData(messages: MessageDbRow[]): Promise<MessageDbRow[]> {
+        if (!messages.length) return [];
+
+        return await Promise.all(messages.map(async message => {
+            if (message.attachments?.trim() && message.pipelineAudit?.trim()) return message;
+
+            const [attachments, audits] = await Promise.all([
+                message.attachments?.trim() ? Promise.resolve(null) : DatabaseManager.getAttachmentsByMessage(message.chatId, message.id),
+                message.pipelineAudit?.trim() ? Promise.resolve(null) : DatabaseManager.getRequestAuditsByMessage(message.chatId, message.id),
+            ]);
+            const normalizedAttachments = attachments ?? [];
+            const normalizedAudits = audits ?? [];
+
+            return {
+                ...message,
+                attachments: message.attachments ?? (normalizedAttachments.length ? JSON.stringify(normalizedAttachments.map(row => attachmentFromRow(row))) : null),
+                pipelineAudit: message.pipelineAudit ?? (normalizedAudits.length ? JSON.stringify(normalizedAudits.map(row => auditFromRow(row))) : null),
+            };
+        }));
+    }
+}
+
+function parsePipelineAudit(value?: string | null): PipelineAuditEvent[] | undefined {
+    if (!value?.trim()) return undefined;
+
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed : undefined;
+    } catch {
+        return undefined;
+    }
+}
+
+function parseAttachments(value?: string | null): StoredAttachment[] | undefined {
+    if (!value?.trim()) return undefined;
+
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed : undefined;
+    } catch {
+        return undefined;
+    }
+}
+
+function attachmentFromRow(row: Awaited<ReturnType<typeof DatabaseManager.getAttachmentsByMessage>>[number]): StoredAttachment {
+    return {
+        kind: row.kind as StoredAttachment["kind"],
+        fileId: row.fileId,
+        fileUniqueId: row.fileUniqueId ?? undefined,
+        fileName: row.fileName,
+        mimeType: row.mimeType ?? undefined,
+        cachePath: row.cachePath,
+        sizeBytes: row.sizeBytes ?? undefined,
+        sha256: row.sha256 ?? undefined,
+        scope: row.scope as StoredAttachment["scope"] | undefined,
+        artifactKind: row.artifactKind as StoredAttachment["artifactKind"] | undefined,
+        metadata: parseJsonObject(row.metadata),
+    };
+}
+
+function auditFromRow(row: Awaited<ReturnType<typeof DatabaseManager.getRequestAuditsByMessage>>[number]): NonNullable<StoredMessage["pipelineAudit"]>[number] {
+    return {
+        stage: row.stage as NonNullable<StoredMessage["pipelineAudit"]>[number]["stage"],
+        status: row.status as NonNullable<StoredMessage["pipelineAudit"]>[number]["status"],
+        startedAt: row.startedAt ?? undefined,
+        finishedAt: row.finishedAt ?? undefined,
+        durationMs: row.durationMs ?? undefined,
+        provider: row.provider as NonNullable<StoredMessage["pipelineAudit"]>[number]["provider"],
+        model: row.model ?? undefined,
+        details: parseJsonObject(row.details),
+        error: row.error ?? undefined,
+    };
+}
+
+function parseJsonObject(value?: string | null): Record<string, unknown> | undefined {
+    if (!value?.trim()) return undefined;
+
+    try {
+        const parsed = JSON.parse(value);
+        return typeof parsed === "object" && parsed !== null ? parsed as Record<string, unknown> : undefined;
+    } catch {
+        return undefined;
+    }
+}
@@ -1,24 +0,0 @@
-import {int, sqliteTable, text} from "drizzle-orm/sqlite-core";
-
-export const messagesTable = sqliteTable("messages", {
-    id: int().primaryKey().unique().notNull(),
-    chatId: int().notNull(),
-    replyToMessageId: int(),
-    fromId: int().notNull(),
-    text: text(),
-    date: int().notNull(),
-    photoMaxSizeFilePath: text(),
-});
-
-export type MessageInsert = typeof messagesTable.$inferInsert;
-
-export const usersTable = sqliteTable("users", {
-    id: int().primaryKey().unique().notNull(),
-    isBot: int().notNull(),
-    firstName: text().notNull(),
-    lastName: text(),
-    userName: text(),
-    isPremium: int(),
-});
-
-export type UserInsert = typeof usersTable.$inferInsert;
@@ -1,24 +1,23 @@
 import {StoredUser} from "../model/stored-user";
 import {Dao} from "../base/dao";
+import {appLogger} from "../logging/logger";
 import {DatabaseManager} from "./database-manager";
-import {UserInsert, usersTable} from "./schema";
-import {eq} from "drizzle-orm";
-import {inArray} from "drizzle-orm/sql/expressions/conditions";
 import {User} from "typescript-telegram-bot-api";
-import {boolToInt, buildExcludedSet} from "../util/utils";
+import {boolToInt} from "../util/utils";
+import {UserDbRow} from "./db-types";

-export class UserDao extends Dao<StoredUser> {
+export class UserDao extends Dao<StoredUser, {id: number}, {ids: number[]}, UserDbRow | UserDbRow[]> {

-    private tag: string = "UserDao";
+    private readonly logger = appLogger.child("dao:users");

    override async getAll(): Promise<StoredUser[]> {
        const then = Date.now();

-        const users = await DatabaseManager.db.select().from(usersTable);
+        const users = await DatabaseManager.getAllUsers();

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getAll()`, `took ${diff}ms; size: ${users.length}`);
+        this.logger.trace("get_all", {dao: "users", duration: `${diff}ms`, size: users.length});

        return this.mapFrom(users);
    }
@@ -26,80 +25,87 @@ export class UserDao extends Dao<StoredUser> {
    override async getById(params: { id: number }): Promise<StoredUser | null> {
        const then = Date.now();

-        const users =
-            await DatabaseManager.db.select()
-                .from(usersTable)
-                .where(
-                    eq(usersTable.id, params.id)
-                );
+        const user = await DatabaseManager.getUserById(params.id);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getById(${params.id})`, `took ${diff}ms; size: ${users.length}`);
+        this.logger.trace("get_by_id", {dao: "users", id: params.id, duration: `${diff}ms`, size: user ? 1 : 0});

-        const u = users[0];
-        if (!u) return null;
-        return this.mapFrom([u])[0];
+        if (!user) return null;
+        return this.mapFrom([user])[0];
    }

    override async getByIds(params: { ids: number[] }): Promise<StoredUser[]> {
        const then = Date.now();

-        const users =
-            await DatabaseManager.db.select()
-                .from(usersTable)
-                .where(
-                    inArray(usersTable.id, params.ids)
-                );
+        const users = await DatabaseManager.getUsersByIds(params.ids);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: getByIds(${params.ids})`, `took ${diff}ms; size: ${users.length}`);
+        this.logger.trace("get_by_ids", {dao: "users", ids: params.ids, duration: `${diff}ms`, size: users.length});

        return this.mapFrom(users);
    }

-    override async insert(values: UserInsert[] | UserInsert): Promise<true> {
+    override async insert(values: UserDbRow[] | UserDbRow): Promise<true> {
        const rows = Array.isArray(values) ? values : [values];
+        if (!rows.length) return true;

        const then = Date.now();
-        const r = await DatabaseManager.db
-            .insert(usersTable)
-            .values(rows)
-            .onConflictDoUpdate({
-                target: usersTable.id,
-                set: buildExcludedSet(usersTable, ["id"])
-            });
+        await DatabaseManager.upsertUsers(rows);

        const now = Date.now();
        const diff = now - then;
-        console.log(`${this.tag}: insert(size: ${rows.length})`, `took ${diff}ms; inserted: ${r.rowsAffected}`);
+        this.logger.debug("insert", {dao: "users", duration: `${diff}ms`, size: rows.length});
        return true;
    }

-    mapTo(users: User[]): UserInsert[] {
+    async updateSettings(
+        id: number,
+        settings: Partial<Pick<StoredUser, "interfaceLanguage" | "aiProvider" | "aiResponseLanguage" | "aiContextSize" | "aiVoiceMode" | "aiImageOutputMode">>
+    ): Promise<true> {
+        await DatabaseManager.updateUserSettings(id, settings);
+
+        return true;
+    }
+
+    mapTo(users: User[]): UserDbRow[] {
        return users.map(u => {
            return {
                id: u.id,
                isBot: boolToInt(u.is_bot),
                firstName: u.first_name,
-                lastName: u.last_name,
-                userName: u.username,
-                isPremium: boolToInt(u.is_premium)
+                lastName: u.last_name ?? null,
+                userName: u.username ?? null,
+                isPremium: boolToInt(u.is_premium),
+                langCode: u.language_code ?? null,
+                interfaceLanguage: null,
+                aiProvider: null,
+                aiResponseLanguage: null,
+                aiContextSize: null,
+                aiVoiceMode: null,
+                aiImageOutputMode: null,
            };
        });
    }

-    mapFrom(users: UserInsert[]): StoredUser[] {
+    mapFrom(users: UserDbRow[]): StoredUser[] {
        return users.map(u => {
            return {
                id: u.id,
                isBot: u.isBot === 1,
                firstName: u.firstName,
-                lastName: u.lastName,
-                userName: u.userName,
-                isPremium: u.isPremium === 1
+                lastName: u.lastName === null ? undefined : u.lastName,
+                userName: u.userName === null ? undefined : u.userName,
+                isPremium: u.isPremium === 1,
+                langCode: u.langCode === null ? undefined : u.langCode,
+                interfaceLanguage: u.interfaceLanguage === null ? undefined : u.interfaceLanguage,
+                aiProvider: u.aiProvider === null ? undefined : u.aiProvider,
+                aiResponseLanguage: u.aiResponseLanguage === null ? undefined : u.aiResponseLanguage,
+                aiContextSize: u.aiContextSize === null ? undefined : u.aiContextSize,
+                aiVoiceMode: u.aiVoiceMode === null ? undefined : u.aiVoiceMode,
+                aiImageOutputMode: u.aiImageOutputMode === null ? undefined : u.aiImageOutputMode,
            };
        });
    }
-}
+}