remix-seo-plus/src/robotstxt/index.ts

Collection of SEO utilities like sitemap, robots.txt, etc. for a Remix application. A fork of https://github.com/balavishnuvj/remix-seo with some added bug fixes and features.

import { RobotsPolicy, RobotsConfig } from "../types";
import { getRobotsText } from "./utils";

const defaultPolicies: RobotsPolicy[] = [
  {
    type: "userAgent",
    value: "*",
  },
  {
    type: "allow",
    value: "/",
  },
];

export async function generateRobotsTxt(
  policies: RobotsPolicy[] = [],
  { appendOnDefaultPolicies = true, headers }: RobotsConfig = {}
) {
  const policiesToUse = appendOnDefaultPolicies
    ? [...defaultPolicies, ...policies]
    : policies;
  const robotText = await getRobotsText(policiesToUse);
  const bytes = new TextEncoder().encode(robotText).byteLength;
  
  return new Response(robotText, {
    headers: {
      ...headers,
      "Content-Type": "text/plain",
      "Content-Length": String(bytes),
    },
  });
}