@mintlify/scraping
Version:
Scrape documentation frameworks to Mintlify docs
151 lines • 7.32 kB
JavaScript
import { getOpenApiTitleAndDescription, optionallyAddLeadingSlash, slugToTitle, } from '@mintlify/common';
import { outputFile } from 'fs-extra';
import fse from 'fs-extra';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import { OpenAPIV3 } from 'openapi-types';
import path, { join, parse, resolve } from 'path';
import { fetchOpenApi } from '../utils/network.js';
export const getOpenApiDefinition = async (pathOrDocumentOrUrl) => {
if (typeof pathOrDocumentOrUrl === 'string') {
if (pathOrDocumentOrUrl.startsWith('http://')) {
// This is an invalid location either for a file or a URL
throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
}
else {
try {
const url = new URL(pathOrDocumentOrUrl);
pathOrDocumentOrUrl = url;
}
catch {
const pathname = path.join(process.cwd(), pathOrDocumentOrUrl.toString());
const file = await fs.readFile(pathname, 'utf-8');
pathOrDocumentOrUrl = yaml.load(file);
}
}
}
const isUrl = pathOrDocumentOrUrl instanceof URL;
if (pathOrDocumentOrUrl instanceof URL) {
if (pathOrDocumentOrUrl.protocol !== 'https:') {
throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
}
pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
}
return { document: pathOrDocumentOrUrl, isUrl };
};
// returns a filename that is unique within the given array of pages
export const generateUniqueFilenameWithoutExtension = (pages, base) => {
let filename = base;
if (pages.includes(filename)) {
let extension = 1;
filename = `${base}-${extension}`;
while (pages.includes(filename)) {
extension += 1;
filename = `${base}-${extension}`;
}
}
return filename;
};
export const createOpenApiFrontmatter = async (filename, openApiMetaTag, version) => {
const data = `---
openapi: ${openApiMetaTag}${version ? `\nversion: ${version}` : ''}
---`;
await outputFile(filename, data);
};
export const prepareStringToBeValidFilename = (str) => str
? str
.replaceAll(' ', '-')
.replace(/\{.*?\}/g, '-') // remove path parameters
.replace(/^-/, '')
.replace(/-$/, '')
.replace(/[{}(),.'\n\/]/g, '') // remove special characters
.replaceAll(/--/g, '-') // replace double hyphens
.toLowerCase()
: undefined;
export function processOpenApiPath(path, pathItemObject, schema, nav, decoratedNav, writePromises, pagesAcc, options,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
findNavGroup) {
const openApiFilePathFromRoot = options.openApiFilePath
? optionallyAddLeadingSlash(options.openApiFilePath)
: undefined;
Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
if (method in pathItemObject) {
const operation = pathItemObject[method];
const groupName = operation?.tags?.[0];
const title = prepareStringToBeValidFilename(operation?.summary) ??
`${method}-${prepareStringToBeValidFilename(path)}`;
const folder = prepareStringToBeValidFilename(groupName) ?? '';
const base = join(options.outDir ?? '', folder, title);
const navGroup = findNavGroup(nav, groupName);
const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
const openapiMetaTag = `${openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''}${method} ${path}`;
const { title: titleTag, description } = getOpenApiTitleAndDescription([
{
filename: options.openApiFilePath
? parse(options.openApiFilePath).name
: 'filler-filename',
spec: schema,
originalFileLocation: options.openApiFilePath,
},
], openapiMetaTag);
navGroup.push(filenameWithoutExtension);
const page = {
openapi: openapiMetaTag,
href: resolve('/', filenameWithoutExtension),
title: titleTag ?? slugToTitle(filenameWithoutExtension),
description,
version: options.version,
};
decoratedNavGroup.push(page);
pagesAcc[filenameWithoutExtension] = page;
const targetPath = options.outDirBasePath
? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
: `${filenameWithoutExtension}.mdx`;
if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, options.version));
}
}
});
}
export const DEFAULT_API_GROUP_NAME = 'API Reference';
export const DEFAULT_WEBHOOK_GROUP_NAME = 'Webhooks';
export function processOpenApiWebhook(webhook, webhookObject, _schema, nav, decoratedNav, writePromises, pagesAcc, options,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
findNavGroup) {
const openApiFilePathFromRoot = options.openApiFilePath
? optionallyAddLeadingSlash(options.openApiFilePath)
: undefined;
Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
if (method in webhookObject) {
const operation = webhookObject[method];
const groupName = operation?.tags?.[0] ?? DEFAULT_WEBHOOK_GROUP_NAME;
const title = prepareStringToBeValidFilename(operation?.summary) ??
`${prepareStringToBeValidFilename(webhook)}`;
const folder = prepareStringToBeValidFilename(groupName) ?? '';
const base = join(options.outDir ?? '', folder, title);
const navGroup = findNavGroup(nav, groupName);
const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
const openapiMetaTag = `${openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''}webhook ${webhook}`;
const description = operation?.description;
navGroup.push(filenameWithoutExtension);
const page = {
openapi: openapiMetaTag,
href: resolve('/', filenameWithoutExtension),
title: slugToTitle(filenameWithoutExtension),
description,
version: options.version,
};
decoratedNavGroup.push(page);
pagesAcc[filenameWithoutExtension] = page;
const targetPath = options.outDirBasePath
? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
: `${filenameWithoutExtension}.mdx`;
if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, options.version));
}
}
});
}
//# sourceMappingURL=common.js.map