@7-docs/cli/dist/fetcher/http.js

Command-line tool for 7-docs to ingest content

export const fetchFiles = async (urls) => {
    return Promise.all(urls.map(async (url) => {
        const _url = new URL(url);
        const response = await fetch(_url);
        if (!response.ok)
            console.error(`${response.status} ${response.statusText}: ${url}`);
        const filePath = _url.pathname.replace(/(\.html?)?$/, '.html');
        return {
            filePath,
            url,
            content: Buffer.from(await response.arrayBuffer())
        };
    }));
};