next-sitemap/dist/esm/builders/robots-txt-builder.js

Sitemap generator for next.js

import { toArray } from '../utils/array.js';
export class RobotsTxtBuilder {
    /**
     * Normalize robots.txt policies
     * @param policies
     * @returns
     */
    normalizePolicy(policies) {
        return policies.map((x) => ({
            ...x,
            allow: toArray(x.allow ?? []),
            disallow: toArray(x.disallow ?? []),
        }));
    }
    /**
     * Add new policy
     * @param key
     * @param rules
     * @returns
     */
    addPolicies(key, rules) {
        return rules.reduce((prev, curr) => `${prev}${key}: ${curr}\n`, '');
    }
    /**
     * Generates robots.txt content
     * @param config
     * @returns
     */
    generateRobotsTxt(config) {
        const { additionalSitemaps, policies } = config.robotsTxtOptions;
        const normalizedPolices = this.normalizePolicy(policies);
        let content = '';
        normalizedPolices.forEach((x) => {
            content += `# ${x.userAgent}\nUser-agent: ${x.userAgent}\n`;
            if (x.allow) {
                content += `${this.addPolicies('Allow', x.allow)}`;
            }
            if (x.disallow) {
                content += `${this.addPolicies('Disallow', x.disallow)}`;
            }
            if (x.crawlDelay) {
                content += `Crawl-delay: ${x.crawlDelay}\n`;
            }
            content += '\n';
        });
        // Append host
        content += `# Host\nHost: ${config.siteUrl}\n`;
        if (additionalSitemaps && additionalSitemaps.length > 0) {
            content += `\n# Sitemaps\n`;
            additionalSitemaps.forEach((x) => {
                content += `Sitemap: ${x}\n`;
            });
        }
        return content;
    }
}