@netronk/mdx-remote/dist/github/index.js

The remote MDX files adapter for Netronk Gen

// src/github/helper/get-files.ts
import path from "path";
import fs from "fs/promises";
import fg from "fast-glob";
import matter from "gray-matter";
import micromatch from "micromatch";

// src/github/api/fetch-tree.ts
async function fetchTree({
  owner,
  repo,
  treeSha,
  recursive = false,
  accessToken,
  init
}) {
  const headers = new Headers(init?.headers);
  headers.set("X-GitHub-Api-Version", "2022-11-28");
  if (accessToken) headers.set("Authorization", `Bearer ${accessToken}`);
  const url = new URL(
    `/repos/${owner}/${repo}/git/trees/${treeSha}`,
    "https://api.github.com"
  );
  if (recursive) url.searchParams.set("recursive", String(recursive));
  const res = await fetch(url, {
    ...init,
    headers
  });
  if (!res.ok)
    throw new Error(`failed to get file tree from GitHub: ${await res.text()}`);
  return await res.json();
}

// src/github/api/fetch-blob.ts
async function fetchBlob({
  url,
  accessToken,
  init
}) {
  const headers = new Headers(init?.headers);
  headers.set("X-GitHub-Api-Version", "2022-11-28");
  if (accessToken) headers.set("Authorization", `Bearer ${accessToken}`);
  const res = await fetch(url, {
    ...init,
    headers
  });
  if (!res.ok)
    throw new Error(
      `failed to get file content from GitHub: ${await res.text()}`
    );
  const blob = await res.json();
  return {
    encoding: "utf8",
    content: Buffer.from(blob.content, blob.encoding).toString("utf8")
  };
}

// src/cache/file-map.ts
function createFileMap() {
  const contentMap = /* @__PURE__ */ new Map();
  return {
    read(path2) {
      return contentMap.get(path2);
    },
    write(path2, content) {
      contentMap.set(path2, content);
    },
    revoke(path2) {
      contentMap.delete(path2);
    }
  };
}

// src/github/helper/get-files.ts
var fileMap = createFileMap();
async function getLocalFiles({
  directory = "./content",
  include = "**/*",
  keepContent = false
}) {
  const files = await fg(include, {
    cwd: path.resolve(directory),
    stats: true
  });
  const virtualFiles = [];
  await Promise.all(
    files.map(async (file) => {
      if (!file.stats) return;
      const hash = file.stats.mtime.getTime();
      const normalized = path.resolve(directory, file.path);
      let content;
      const cached = fileMap.read(normalized);
      if (cached && cached.hash === hash) {
        content = cached.content;
      } else {
        content = await fs.readFile(normalized).then((res) => res.toString());
        if (!content) return;
        fileMap.write(normalized, { hash, content });
      }
      if (path.extname(normalized) === ".json") {
        virtualFiles.push({
          path: file.path,
          type: "meta",
          data: JSON.parse(content)
        });
      } else {
        const { data } = matter(content);
        virtualFiles.push({
          path: file.path,
          type: "page",
          data: {
            ...data,
            data: {
              content: keepContent ? content : void 0,
              resolver: {
                type: "local",
                file: normalized
              }
            }
          }
        });
      }
    })
  );
  return virtualFiles;
}
async function getGitHubFiles({
  owner,
  repo,
  accessToken,
  treeSha = "main",
  include = "**/*",
  directory = "./content",
  keepContent = false
}) {
  if (directory.startsWith("../"))
    throw new Error(`Directory path: ${directory} cannot start with '../'`);
  const filter = directory.startsWith("./") ? directory.slice("./".length) : directory;
  const virtualFiles = [];
  const tree = await fetchTree({
    owner,
    repo,
    accessToken,
    treeSha,
    recursive: true,
    init: {
      cache: "force-cache"
    }
  });
  await Promise.all(
    tree.tree.map(async (file) => {
      if (file.type === "tree" || !file.path.startsWith(filter) || !micromatch.isMatch(file.path, include))
        return;
      const normalizedPath = file.path.slice(filter.length).split("/").filter((s) => s.length > 0).join("/");
      const blob = await fetchBlob({
        url: file.url,
        accessToken,
        init: {
          cache: "force-cache"
        }
      });
      const content = blob.content;
      if (path.extname(file.path) === ".json") {
        virtualFiles.push({
          path: normalizedPath,
          type: "meta",
          data: JSON.parse(content)
        });
      } else {
        const { data } = matter(content);
        virtualFiles.push({
          path: normalizedPath,
          type: "page",
          data: {
            ...data,
            data: {
              content: keepContent ? content : void 0,
              resolver: {
                type: "github",
                blobUrl: file.url,
                accessToken
              }
            }
          }
        });
      }
    })
  );
  return virtualFiles;
}

// src/github/constants.ts
var USE_LOCAL = process.env.NODE_ENV === "development";

// src/github/source.ts
async function createSourceAuto(options) {
  return {
    files: USE_LOCAL ? await getLocalFiles(options) : await getGitHubFiles({
      ...options,
      ...options.github
    })
  };
}

// src/github/helper/resolve-file.ts
import fs2 from "fs/promises";
async function resolveFile(page) {
  const resolver = page.data.data.resolver;
  if (resolver.type === "local") {
    return await fs2.readFile(resolver.file).then((res) => res.toString());
  }
  const blob = await fetchBlob({
    url: resolver.blobUrl,
    accessToken: resolver.accessToken
  });
  return blob.content;
}

// src/github/helper/search-indexes.ts
import { structure } from "@netronk/gen-core/mdx-plugins";
import {
  createGetUrl,
  getSlugs,
  parseFilePath
} from "@netronk/gen-core/source";
async function buildSearchIndexes(options) {
  const {
    slug = getSlugs,
    baseUrl = "/",
    url = createGetUrl(baseUrl),
    ...rest
  } = options;
  const files = await getLocalFiles({
    ...rest,
    keepContent: true
  });
  const output = [];
  files.forEach((file) => {
    if (file.type !== "page" || !file.data.data.content) return;
    const info = parseFilePath(file.path);
    const structuredData = structure(file.data.data.content);
    output.push({
      title: file.data.title,
      id: file.path,
      structuredData,
      url: url(slug(info), info.locale)
    });
  });
  return output;
}
export {
  buildSearchIndexes,
  createSourceAuto,
  fetchBlob,
  fetchTree,
  getGitHubFiles,
  getLocalFiles,
  resolveFile
};