auron/src/controllers/metric.ts

Interact with your ATProto labeler from your terminal

import EventEmitter from "events";
import { withLoader } from "../utils/loader";
import { database } from "../services/db";
import { writeFile } from "node:fs/promises";

type ActionTimeMetric = {
  item: number;
  time: number;
};

type ActionTimeMetricGroup = {
  human: ActionTimeMetric;
  automated: ActionTimeMetric;
  unhandled: number;
};

const automods = [
  "did:plc:fokhsjjl5opb2hrkcmc2swfi",
  "did:plc:znoll2wxc7jt736ulwzlxiqk",
  "did:plc:ar7c4by46qjdydhdevvrndac",
];

// Define histogram buckets (time in minutes)
const timeBins = [
  { label: "0-10 secs", min: 0, max: 10 },
  { label: "10-30 secs", min: 10, max: 30 },
  { label: "30-60 secs", min: 30, max: 60 },
  { label: "1-5 mins", min: 60, max: 300 },
  { label: "5-10 mins", min: 300, max: 600 },
  { label: "10-30 mins", min: 600, max: 1800 },
  { label: "30-60 mins", min: 1800, max: 3600 },
  { label: "1-3 hrs", min: 3600, max: 10800 },
  { label: "3-6 hrs", min: 10800, max: 21600 },
  { label: "6-12 hrs", min: 21600, max: 43200 },
  { label: "12-24 hrs", min: 43200, max: 86400 },
  { label: "1-2 days", min: 86400, max: 172800 },
  { label: "2-3 days", min: 172800, max: 259200 },
  { label: "3-4 days", min: 259200, max: 345600 },
  { label: "4-5 days", min: 345600, max: 432000 },
  { label: "5-7 days", min: 432000, max: 604800 },
  { label: "7-10 days", min: 604800, max: 864000 },
  { label: "10-15 days", min: 864000, max: 1296000 },
  { label: "15-20 days", min: 1296000, max: 1728000 },
];

export const computeActionTime = async (options: {
  start?: string;
  end?: string;
  file?: string;
}) => {
  const emitter = new EventEmitter();
  const allEvents = await withLoader(
    `Computing action time...`,
    async (updateMessage) => {
      const startedAt = new Date().getTime();
      emitter.on("update", ({ message }) => {
        updateMessage(message);
      });

      const events = await database.getEvents({
        cursor: 0,
        createdAfter: options.start,
        createdBefore: options.end,
        types: [
          "tools.ozone.moderation.defs#modEventReport",
          "tools.ozone.moderation.defs#modEventAcknowledge",
          "tools.ozone.moderation.defs#modEventTakeDown",
          "tools.ozone.moderation.defs#modEventLabel",
        ],
      });

      const metrics: Record<string, ActionTimeMetricGroup> = {
        account: {
          human: { item: 0, time: 0 },
          automated: { item: 0, time: 0 },
          unhandled: 0,
        },
        record: {
          human: { item: 0, time: 0 },
          automated: { item: 0, time: 0 },
          unhandled: 0,
        },
      };

      // Initialize separate histograms for human and automated actions
      const humanHandledRecords: Record<string, number> = {};
      const automatedHandledRecords: Record<string, number> = {};
      const humanHandledAccounts: Record<string, number> = {};
      const automatedHandledAccounts: Record<string, number> = {};

      for (const bin of timeBins) {
        humanHandledRecords[bin.max] = 0;
        automatedHandledRecords[bin.max] = 0;
        humanHandledAccounts[bin.max] = 0;
        automatedHandledAccounts[bin.max] = 0;
      }

      let i = 0;
      const subjectReportTime = new Map<string, string>();
      const unhandledSubjects = new Set<string>();
      let oldestReportTime = new Date().toISOString();
      let latestReportTime = new Date(0).toISOString();

      for (const event of events) {
        const isRecord = !!event.subjectUri;
        const subject = event.subjectUri || event.subjectDid;
        const reportTime = subjectReportTime.get(subject);

        if (event.action === "tools.ozone.moderation.defs#modEventReport") {
          if (!reportTime) {
            subjectReportTime.set(subject, event.createdAt);
          }
          if (new Date(event.createdAt) < new Date(oldestReportTime)) {
            oldestReportTime = event.createdAt;
          }
          if (new Date(event.createdAt) > new Date(latestReportTime)) {
            latestReportTime = event.createdAt;
          }
          unhandledSubjects.add(subject);
          continue;
        }

        if (!reportTime) {
          continue;
        }

        const isAutomated = automods.includes(event.createdBy);
        const timeDiffMs =
          new Date(event.createdAt).getTime() - new Date(reportTime).getTime();
        const timeDiffMins = timeDiffMs / 1000; // Convert ms to minutes

        // Assign to histogram bins
        const targetHistogram = isRecord
          ? isAutomated
            ? automatedHandledRecords
            : humanHandledRecords
          : isAutomated
          ? automatedHandledAccounts
          : humanHandledAccounts;

        for (const bin of timeBins) {
          // if (bin.max === 10) {
          //   console.log(event)
          // }
          if (timeDiffMins >= bin.min && timeDiffMins < bin.max) {
            targetHistogram[bin.max]++;
            break;
          }
        }

        // Store into metrics
        if (isRecord) {
          if (isAutomated) {
            metrics.record.automated.item++;
            metrics.record.automated.time += timeDiffMs;
          } else {
            metrics.record.human.item++;
            metrics.record.human.time += timeDiffMs;
          }
        } else {
          if (isAutomated) {
            metrics.account.automated.item++;
            metrics.account.automated.time += timeDiffMs;
          } else {
            metrics.account.human.item++;
            metrics.account.human.time += timeDiffMs;
          }
        }

        unhandledSubjects.delete(subject);
        subjectReportTime.delete(subject);

        i++;
        if (i % 10000 === 0) {
          const timeSpent = parseInt(
            `${(new Date().getTime() - startedAt) / 1000}`
          );
          emitter.emit("update", {
            message: `Processed ${i} out of ${events.length} event chunks. Time spent: ${timeSpent}s`,
          });
          await new Promise((resolve) => setTimeout(resolve, 1000));
        }
      }

      console.log(
        `\nReport time range: ${oldestReportTime} - ${latestReportTime}`
      );

      if (unhandledSubjects.size > 0) {
        for (const unhandled of unhandledSubjects) {
          if (unhandled.startsWith("did:")) {
            metrics.account.unhandled++;
          } else {
            metrics.record.unhandled++;
          }
        }
      }

      if (options.file) {
        await writeFile(
          options.file,
          JSON.stringify(
            {
              metrics,
              humanHandledRecords,
              automatedHandledRecords,
              humanHandledAccounts,
              automatedHandledAccounts,
              oldestReportTime,
              latestReportTime,
            },
            null,
            2
          ),
          "utf-8"
        );
      } else {
        console.log(`\n Human-Handled Action Time Histogram (Records):`);
        console.table(humanHandledRecords);
        console.log(`\n Automated-Handled Action Time Histogram (Records):`);
        console.table(automatedHandledRecords);
        console.log(`\n Human-Handled Action Time Histogram (Accounts):`);
        console.table(humanHandledAccounts);
        console.log(`\n Automated-Handled Action Time Histogram (Accounts):`);
        console.table(automatedHandledAccounts);
      }
    }
  );

  return allEvents;
};