@guild-docs/algolia
Version:
362 lines (361 loc) • 15.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.docusaurusToRoutes = exports.indexToAlgolia = void 0;
/* eslint-disable @typescript-eslint/no-non-null-assertion, no-else-return */
const promises_1 = require("node:fs/promises");
const node_fs_1 = require("node:fs");
const sortBy_js_1 = __importDefault(require("lodash/sortBy.js"));
const isString_js_1 = __importDefault(require("lodash/isString.js"));
const isArray_js_1 = __importDefault(require("lodash/isArray.js"));
const flatten_js_1 = __importDefault(require("lodash/flatten.js"));
const compact_js_1 = __importDefault(require("lodash/compact.js"));
const map_js_1 = __importDefault(require("lodash/map.js"));
const identity_js_1 = __importDefault(require("lodash/identity.js"));
const github_slugger_1 = __importDefault(require("github-slugger"));
const remove_markdown_1 = __importDefault(require("remove-markdown"));
const algoliasearch_1 = __importDefault(require("algoliasearch"));
const gray_matter_1 = __importDefault(require("gray-matter"));
const glob_1 = __importDefault(require("glob"));
const extractToC = (content) => {
const slugger = new github_slugger_1.default();
const lines = content.split('\n');
let isCodeBlock = false;
let currentDepth = 0;
let currentParent;
const slugs = lines.reduce((acum, value) => {
var _a, _b;
if (value.match(/^```(.*)/)) {
if (isCodeBlock) {
isCodeBlock = false;
}
else {
isCodeBlock = true;
return acum;
}
}
else if (isCodeBlock) {
return acum;
}
const result = value.match(/(##+ )(.+)/);
if (!result)
return acum;
const depth = ((_a = result[1]) === null || _a === void 0 ? void 0 : _a.length) - 3;
if (depth > 1) {
return acum;
}
const heading = (_b = result[2]) === null || _b === void 0 ? void 0 : _b.trim();
const record = {
children: [],
title: heading,
anchor: slugger.slug(heading),
};
if (depth > 0) {
currentParent === null || currentParent === void 0 ? void 0 : currentParent.children.push(record);
if (depth > currentDepth) {
currentParent = record;
}
}
else {
currentParent = record;
acum.push(record);
}
currentDepth = depth;
return acum;
}, []);
return slugs;
};
const normalizeDomain = (domain) => (domain.endsWith('/') ? domain : `${domain}`);
const contentForRecord = (content) => {
let isCodeBlock = false;
let isMeta = false;
return (0, remove_markdown_1.default)(content
.split('\n')
.map(line => {
// remove code snippets
if (line.match(/^```(.*)/)) {
if (isCodeBlock) {
isCodeBlock = false;
return null;
}
else {
isCodeBlock = true;
return null;
}
}
else if (isCodeBlock) {
return null;
}
// remove metadata headers
if (line.startsWith('---')) {
if (isMeta) {
isMeta = false;
return null;
}
else {
isMeta = true;
return null;
}
}
else if (isMeta) {
return null;
}
// remove titles
if (line.startsWith('#')) {
return null;
}
// remove `import` and `export`
if (!isCodeBlock && (line.match(/^export(.*)/) || line.match(/^import(.*)/))) {
return null;
}
return line;
})
.filter(line => line !== null)
.join(' '));
};
async function routesToAlgoliaRecords(routes, source, domain, mdx = true, objectsPrefix = new github_slugger_1.default().slug(source), parentRoute) {
const objects = [];
async function routeToAlgoliaRecords(topPath, parentLevelName, slug, title) {
if (!slug) {
return;
}
const fileContent = await (0, promises_1.readFile)(`./${(0, compact_js_1.default)([parentRoute === null || parentRoute === void 0 ? void 0 : parentRoute.path, topPath, slug]).join('/')}.md${mdx ? 'x' : ''}`);
const { data: meta, content } = (0, gray_matter_1.default)(fileContent.toString());
const resolvedTitle = title || meta.title || meta.sidebar_label;
if (!resolvedTitle) {
return;
}
const toc = extractToC(content);
objects.push({
objectID: `${objectsPrefix}-${slug}`,
headings: toc.map(t => t.title),
toc,
content: contentForRecord(content),
url: `${domain}${(0, compact_js_1.default)([parentRoute === null || parentRoute === void 0 ? void 0 : parentRoute.path, topPath, slug]).join('/')}`,
domain,
hierarchy: (0, compact_js_1.default)([source, parentRoute === null || parentRoute === void 0 ? void 0 : parentRoute.$name, parentLevelName, resolvedTitle]),
source,
title: resolvedTitle,
type: meta.type || 'Documentation',
});
}
await Promise.all((0, map_js_1.default)(routes._, async (topRoute, topPath) => {
if (!topRoute) {
return Promise.resolve();
}
if ((0, isString_js_1.default)(topRoute)) {
console.warn(`ignored ${topRoute}`);
return Promise.resolve();
}
else if ((0, isArray_js_1.default)(topRoute)) {
console.warn(`ignored ${topRoute}`);
return Promise.resolve();
}
else {
if (topRoute.$name && !topRoute.$routes) {
return await routeToAlgoliaRecords(undefined, undefined, topPath, topRoute.$name);
}
else {
return await Promise.all((0, map_js_1.default)(topRoute.$routes, route => {
if ((0, isArray_js_1.default)(route)) {
// `route` is `['slug', 'title']`
return routeToAlgoliaRecords(topPath, topRoute.$name, route[0], route[1]);
}
else {
// `route` is `'slug'`
if (route.startsWith('$')) {
const refName = route.substring(1);
const refs = topRoute._;
const subRoutes = refs[refName];
if (subRoutes) {
return new Promise(resolve => {
routesToAlgoliaRecords({
_: {
[refName]: subRoutes,
},
}, source, domain, mdx, new github_slugger_1.default().slug(`${source}-${refName}`), {
$name: topRoute.$name,
path: topPath,
}).then(objs => {
objects.push(...objs);
resolve();
});
});
}
else {
console.warn(`could not find routes for reference ${route}`);
}
}
else {
return routeToAlgoliaRecords(topPath, topRoute.$name, route);
}
}
}));
}
}
}));
return objects;
}
async function pluginsToAlgoliaRecords(
// TODO: fix later
plugins, source, domain, objectsPrefix = new github_slugger_1.default().slug(source)) {
const objects = [];
const slugger = new github_slugger_1.default();
plugins.forEach((plugin) => {
const toc = extractToC(plugin.readme || '');
objects.push({
objectID: slugger.slug(`${objectsPrefix}-${plugin.title}`),
headings: toc.map(t => t.title),
toc,
content: contentForRecord(plugin.readme || ''),
url: `${domain}plugins/${plugin.identifier}`,
domain,
hierarchy: [source, 'Plugins'],
source,
title: plugin.title,
type: 'Plugin',
});
});
return objects;
}
async function nextraToAlgoliaRecords({ docsBaseDir }, source, domain, objectsPrefix = new github_slugger_1.default().slug(source)) {
return new Promise((resolve, reject) => {
const objects = [];
const slugger = new github_slugger_1.default();
// cache for all needed `meta.json` files
const metadataCache = {};
const getMetaFromFile = (path) => {
if ((0, node_fs_1.statSync)(path)) {
return JSON.parse((0, node_fs_1.readFileSync)(path).toString() || '{}');
}
return {};
};
const getMetadataForFile = (filePath) => {
const hierarchy = [];
const fileDir = filePath.split('/').slice(0, -1).join('/');
const fileName = filePath.split('/').pop();
const folders = filePath.replace(docsBaseDir, '').replace(fileName, '').split('/').filter(Boolean);
// docs/guides/advanced -> ['Guides', 'Advanced']
// by reading meta from:
// - docs/guides/meta.json (for 'advanced' folder)
// - docs/meta.json (for 'guides' folder)
while (folders.length) {
const folder = folders.pop();
const path = folders.join('/');
if (!metadataCache[path]) {
metadataCache[path] = getMetaFromFile(`${docsBaseDir}${docsBaseDir.endsWith('/') ? '' : '/'}${path}/meta.json`);
}
const folderName = metadataCache[path][folder];
const resolvedFolderName = typeof folderName === 'string' ? folderName : (folderName === null || folderName === void 0 ? void 0 : folderName.title) || folder;
if (resolvedFolderName) {
hierarchy.unshift(resolvedFolderName);
}
}
if (!metadataCache[fileDir]) {
metadataCache[fileDir] = getMetaFromFile(`${fileDir}${fileDir.endsWith('/') ? '' : '/'}meta.json`);
}
const title = metadataCache[fileDir][fileName.replace('.mdx', '')];
const resolvedTitle = typeof title === 'string' ? title : title === null || title === void 0 ? void 0 : title.title;
const urlPath = filePath.replace(docsBaseDir, '').replace(fileName, '').split('/').filter(Boolean).join('/');
return [resolvedTitle || fileName.replace('.mdx', ''), hierarchy, urlPath];
};
(0, glob_1.default)(`${docsBaseDir}${docsBaseDir.endsWith('/') ? '' : '/'}**/*.mdx`, (err, files) => {
if (err) {
reject(err);
}
else {
files.forEach(file => {
var _a;
// eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
const filename = (_a = file.split('/').pop()) === null || _a === void 0 ? void 0 : _a.split('.')[0];
const fileContent = (0, node_fs_1.readFileSync)(file);
const { data: meta, content } = (0, gray_matter_1.default)(fileContent.toString());
const toc = extractToC(content);
const [title, hierarchy, urlPath] = getMetadataForFile(file);
objects.push({
objectID: slugger.slug(`${objectsPrefix}-${[...hierarchy, filename].join('-')}`),
headings: toc.map(t => t.title),
toc,
content: contentForRecord(content),
url: `${domain}${urlPath}/${filename}`,
domain,
hierarchy,
source,
title,
type: meta.type || 'Documentation',
});
});
resolve(objects);
}
});
});
}
const indexToAlgolia = async ({ routes: routesArr, docusaurus, plugins = [], source, domain, nextra, postProcessor = identity_js_1.default,
// TODO: add `force` flag
dryMode = true, lockfilePath, }) => {
const normalizedRoutes = docusaurus ? [(0, exports.docusaurusToRoutes)(docusaurus)] : routesArr || [];
const objects = postProcessor([
...(0, flatten_js_1.default)(await Promise.all(normalizedRoutes.map(routes => routesToAlgoliaRecords(routes, source, normalizeDomain(domain), !docusaurus)))),
...(await pluginsToAlgoliaRecords(plugins, source, normalizeDomain(domain))),
...(nextra ? await nextraToAlgoliaRecords(nextra, source, normalizeDomain(domain)) : []),
]);
const recordsAsString = JSON.stringify((0, sortBy_js_1.default)(objects, 'objectID'), (key, value) => (key === 'content' ? '-' : value), 2);
const lockFileExists = (0, node_fs_1.existsSync)(lockfilePath);
const lockfileContent = JSON.stringify((0, sortBy_js_1.default)(JSON.parse(lockFileExists ? (0, node_fs_1.readFileSync)(lockfilePath, 'utf-8') : '[]'), 'objectID'), (key, value) => (key === 'content' ? '-' : value), 2);
if (dryMode) {
console.log(`${lockfilePath} updated!`);
(0, node_fs_1.writeFileSync)(lockfilePath, recordsAsString);
}
else {
if (!lockFileExists || recordsAsString !== lockfileContent) {
if (['ALGOLIA_APP_ID', 'ALGOLIA_ADMIN_API_KEY', 'ALGOLIA_INDEX_NAME'].some(envVar => !process.env[envVar])) {
console.error('Some Algolia environment variables are missing!');
return;
}
if (lockFileExists) {
console.log('changes detected, updating Algolia index!');
}
else {
console.log('no lockfile detected, push all records');
}
const client = (0, algoliasearch_1.default)(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY);
const index = client.initIndex(process.env.ALGOLIA_INDEX_NAME);
index
.deleteBy({
filters: `source: "${source}"`,
})
.then(() => index.saveObjects(objects))
.then(({ objectIDs }) => {
console.log(objectIDs);
})
.catch(console.error);
(0, node_fs_1.writeFileSync)(lockfilePath, recordsAsString);
}
}
};
exports.indexToAlgolia = indexToAlgolia;
const docusaurusToRoutes = ({ sidebars }) => {
const routes = { _: {} };
(0, map_js_1.default)(sidebars.docs, (children, title) => {
var _a;
if (children.every(c => c.includes('/'))) {
const path = `docs/${children[0].split('/')[0]}`;
routes._[path] = {
$name: title,
$routes: [...children],
};
}
else {
if (routes._.docs) {
(_a = routes._.docs.$routes) === null || _a === void 0 ? void 0 : _a.push(...children);
}
else {
routes._.docs = { $routes: [...children] };
}
}
});
return routes;
};
exports.docusaurusToRoutes = docusaurusToRoutes;