auron/src/services/file.ts

Interact with your ATProto labeler from your terminal

import fs from "node:fs/promises";
import { Parser } from "json2csv";
import { chunkArray } from "@atproto/common";

export const writeJsonFile = async (
  filename: string,
  content: Array<unknown>
) => {
  // When writing json that's too big, JSON.stringify can throw a RangeError: "Invalid string length"
  // To avoid this, we can write the content in chunks of 5K items at a time
  // Since we are appending to a file, let's make sure we empty the file first
  await fs.writeFile(filename, "[");
  let i = 0;
  for (const chunk of chunkArray(content, 5000)) {
    await fs.appendFile(
      filename,
      i > 0 ? ",\n" : "\n" + JSON.stringify(chunk).slice(1, -1)
    );
    i++;
  }
  await fs.appendFile(filename, "]");
  return;
};

export const writeCsvFile = async (
  filename: string,
  content: Array<unknown>
) => {
  const json2csvParser = new Parser();
  const csv = json2csvParser.parse(content);
  return fs.writeFile(filename, csv);
};