ap-ssg/lib/utils/generateRobotsTxt.js

A fast, modular, SEO-optimized static site generator that minifies CSS, JS, and HTML for improved performance. It also supports JSON-LD, sitemap generation, and more, making it ideal for production-ready websites.

const fs = require("fs-extra");
const {
  getSrcRobotsTxtFilePath,
  getBuildRobotsTxtFilePath,
} = require("../configs/paths");

/**
 * Accept sitemapUrl and add to robots.txt file and generate
 * @param sitemapUrl
 * @returns {Promise<string>}
 */
exports.generateRobotsTxt = async function (sitemapUrl) {
  const robotsContent = (await getRobotsTxtContent()).trim();
  let robotsTxtData =
    robotsContent ||
    "# Generated by ap-ssg (Static Site Generator)\n" +
      "# Allow all web crawlers to access all pages\n" +
      "User-agent: *\n" +
      "Disallow:\n";

  robotsTxtData += `\nSitemap: ${sitemapUrl}\n`;

  const outputPath = getBuildRobotsTxtFilePath();
  await fs.writeFile(outputPath, robotsTxtData);

  return `robots.txt has been generated at: ${outputPath}`;
};

async function getRobotsTxtContent() {
  try {
    const filePath = getSrcRobotsTxtFilePath();
    const fileExists = await fs.pathExists(filePath);

    if (fileExists) {
      return (await fs.readFile(filePath, "utf8"))
        .split("\n")
        .filter((line) => !line.startsWith("Sitemap:"))
        .join("\n");
    }
    return "";
  } catch {
    console.log("Error getting robots.txt data");
    return "";
  }
}