ap-ssg/lib/utils/generateRobotsTxt.js

Static Site Generator

const fs = require("fs-extra");
const path = require("path");

/**
 * Accept sitemapUrl and add to robots.txt file and generate
 * @param sitemapUrl
 * @returns {Promise<string>}
 */
exports.generateRobotsTxt = async function (sitemapUrl) {
  const robotsContent = (await getRobotsTxtContent()).trim();
  let robotsTxtData =
    robotsContent ||
    "# Generated by ap-ssg (Static Site Generator)\n" +
      "# Allow all web crawlers to access all pages\n" +
      "User-agent: *\n" +
      "Disallow:\n";

  robotsTxtData += `\nSitemap: ${sitemapUrl}\n`;

  const buildDir = path.join(process.cwd(), "build");
  const robotsTxtPath = path.join(buildDir, "robots.txt");

  await fs.ensureDir(buildDir);
  await fs.writeFile(robotsTxtPath, robotsTxtData);

  return `robots.txt has been generated at: ${robotsTxtPath}`;
};

async function getRobotsTxtContent() {
  try {
    const filePath = path.join(process.cwd(), "src", "assets", "robots.txt");
    if (fs.existsSync(filePath)) {
      return (await fs.readFile(filePath, "utf8"))
        .split("\n")
        .filter((line) => !line.startsWith("Sitemap:"))
        .join("\n");
    }
    return "";
  } catch {
    console.log("Error getting robots.txt data");
    return "";
  }
}