spido
Version:
Web crawler/spider for node.js & nest.js server.
58 lines (57 loc) • 2.11 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.writeSitemap = exports.addLinksToXML = exports.sitemapLinksGenerator = void 0;
const fs_1 = __importDefault(require("fs"));
const xml_formatter_1 = __importDefault(require("xml-formatter"));
const utils_1 = require("./core/utils");
// xml sitemap generator using crawler.cjs
const sitemapLinksGenerator = async (url) => {
const utils = new utils_1.Utils();
const response = await utils.getResponse(url);
const responseData = response?.response.data;
const internalLinks = await utils.getInternalLinks(responseData);
if (internalLinks) {
const sitemap = internalLinks.map((link) => {
return `<url><loc>${link}</loc></url>`;
});
return sitemap;
}
};
exports.sitemapLinksGenerator = sitemapLinksGenerator;
// convert sitemapGenerator to xml
const addLinksToXML = async (sitemap) => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${sitemap.join("\n")}
</urlset>`;
const formattedXML = (0, xml_formatter_1.default)(xml);
return formattedXML;
};
exports.addLinksToXML = addLinksToXML;
// write sitemap to file
const writeSitemap = async (xml, path) => {
const file = "sitemap.xml";
//write file to current directory or any other directory
//if user want to write file to another directory then change the path
if (!path) {
fs_1.default.writeFile(file, xml, (err) => {
if (err)
throw err;
console.log("Sitemap generated successfully");
});
return file;
}
else if (path) {
console.log(path);
fs_1.default.writeFile(`${path}${file}`, xml, (err) => {
if (err)
throw err;
console.log("Sitemap generated successfully");
});
return path + file;
}
};
exports.writeSitemap = writeSitemap;