auron/dist/controllers/queue.js

Interact with your ATProto labeler from your terminal

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.processSubjects = exports.fetchRecordsForSubjects = exports.fetchReposForSubjects = exports.saveQueueItems = exports.fetchQueueItems = void 0;
const events_1 = __importDefault(require("events"));
const loader_1 = require("../utils/loader");
const atproto_1 = require("../api/atproto");
const db_1 = require("../services/db");
const transformers_1 = require("../services/subject/transformers");
const common_1 = require("@atproto/common");
const fetchQueueItems = async (options) => {
    const emitter = new events_1.default();
    const subjects = await (0, loader_1.withLoader)(`Fetching queue items...`, async (updateMessage) => {
        emitter.on("update", ({ nextCursor, subjectCount, maxCount }) => {
            updateMessage(`Fetched page with cursor: ${nextCursor}, total subjects: ${subjectCount} out of ${maxCount}`);
        });
        const { cursor, subjectStatuses } = await (0, atproto_1.getQueueItems)({
            maxCount: options.count ? Number(options.count) : undefined,
            cursor: options.cursor,
        }, emitter);
        updateMessage(`Fetched ${subjectStatuses.length} queue items. Cursor: ${cursor}`);
        return subjectStatuses;
    });
    return subjects;
};
exports.fetchQueueItems = fetchQueueItems;
const saveQueueItems = async (subjects) => {
    await (0, loader_1.withLoader)(`Storing ${subjects.length} queue items...`, async (updateMessage) => {
        if (subjects.length === 0) {
            updateMessage("No queue items to store");
            throw new Error("No queue items to store");
        }
        for (const chunk of (0, common_1.chunkArray)(subjects, 500)) {
            await db_1.database.insertSubjects(chunk.map(transformers_1.transformStatusToSubject));
            updateMessage(`Stored ${chunk.length} out of ${subjects.length} queue items in local database`);
        }
        updateMessage(`Stored ${subjects.length} queue items in local database`);
    });
};
exports.saveQueueItems = saveQueueItems;
const fetchReposForSubjects = async () => {
    const emitter = new events_1.default();
    const subjects = await (0, loader_1.withLoader)(`Fetching repos for subjects...`, async (updateMessage) => {
        emitter.on("update", ({ total, repoCount }) => {
            updateMessage(`Fetched ${repoCount} repos out of ${total}`);
        });
        const dids = await db_1.database.getMissingRepoDids();
        updateMessage(`Found ${dids.length} missing repos to be fetched`);
        if (dids.length === 0) {
            return;
        }
        const repos = await (0, atproto_1.getRepos)({ dids }, emitter);
        updateMessage(`Fetched ${repos.length} repos. Saving in local database now`);
        for (const chunk of (0, common_1.chunkArray)(repos, 500)) {
            await db_1.database.saveRepos(chunk.map(transformers_1.transformRepoViewToRepo));
            updateMessage(`Saved ${chunk.length} out of ${repos.length} repos in local database now`);
        }
        updateMessage(`Saved ${repos.length} repos in local database`);
        return;
    });
    return subjects;
};
exports.fetchReposForSubjects = fetchReposForSubjects;
const fetchRecordsForSubjects = async () => {
    const emitter = new events_1.default();
    const subjects = await (0, loader_1.withLoader)(`Fetching records for subjects...`, async (updateMessage) => {
        emitter.on("update", ({ total, recordCount }) => {
            updateMessage(`Fetched ${recordCount} records out of ${total}`);
        });
        const uris = await db_1.database.getMissingRecordUris();
        updateMessage(`Found ${uris.length} missing records to be fetched`);
        if (uris.length === 0) {
            return;
        }
        const records = await (0, atproto_1.getRecords)({ uris }, emitter);
        updateMessage(`Fetched ${records.length} records. Saving in local database now`);
        for (const chunk of (0, common_1.chunkArray)(records, 500)) {
            await db_1.database.saveRecords(chunk.map(transformers_1.transformRecordViewToRecord));
            updateMessage(`Saved ${chunk.length} out of ${records.length} records in local database`);
        }
        updateMessage(`Saved ${records.length} records in local database`);
        return;
    });
    return subjects;
};
exports.fetchRecordsForSubjects = fetchRecordsForSubjects;
const processSubjects = async (options) => {
    let subjects = await db_1.database.listSubjects({
        subjectType: options.type,
        cursor: options.cursor,
        limit: options.count,
    });
    if (options.bio || options.keyword) {
        const keyword = `${options.bio || options.keyword}`.toLowerCase();
        subjects = subjects.filter((subject) => {
            const profile = JSON.parse(subject.profile || "{}");
            return profile.description?.toLowerCase().includes(keyword);
        });
    }
    if (options.keyword) {
        const keyword = options.keyword.toLowerCase();
        subjects = subjects.filter((subject) => {
            const recordValue = JSON.parse(subject.value || "{}");
            return recordValue.text?.toLowerCase().includes(keyword);
        });
    }
    return subjects;
};
exports.processSubjects = processSubjects;