mcard-js/dist/chunk-2MRBTM72.js

MCard - Content-addressable storage with cryptographic hashing, handle resolution, and vector search for Node.js and browsers

import {
  ContentTypeInterpreter,
  MCard
} from "./chunk-PW4XS7M3.js";
import {
  __export
} from "./chunk-MLKGABMK.js";

// src/Loader.ts
var Loader_exports = {};
__export(Loader_exports, {
  loadFileToCollection: () => loadFileToCollection,
  processAndStoreFile: () => processAndStoreFile
});
import * as fs2 from "fs/promises";
import * as path2 from "path";

// src/FileIO.ts
var FileIO_exports = {};
__export(FileIO_exports, {
  isProblematicFile: () => isProblematicFile,
  listFiles: () => listFiles,
  processFileContent: () => processFileContent,
  readFileSafely: () => readFileSafely,
  streamReadNormalizedText: () => streamReadNormalizedText
});
import * as crypto from "crypto";
import * as fs from "fs/promises";
import * as path from "path";
async function streamReadNormalizedText(filePath, options) {
  const { byteCap, wrapWidth } = options;
  const sha = crypto.createHash("sha256");
  let totalSize = 0;
  let producedText = "";
  let currentLen = 0;
  const handle = await fs.open(filePath, "r");
  try {
    const buffer = new Uint8Array(8192);
    let remaining = byteCap;
    const decoder = new TextDecoder("utf-8", { fatal: false });
    let position = 0;
    while (remaining > 0) {
      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
      if (bytesRead === 0) break;
      position += bytesRead;
      const chunk = buffer.subarray(0, bytesRead);
      sha.update(chunk);
      totalSize += bytesRead;
      remaining -= bytesRead;
      const s2 = decoder.decode(chunk, { stream: true });
      for (const ch of s2) {
        if (ch === "\r") continue;
        producedText += ch;
        if (ch === "\n") {
          currentLen = 0;
        } else {
          currentLen++;
          if (wrapWidth > 0 && currentLen >= wrapWidth) {
            producedText += "\n";
            currentLen = 0;
          }
        }
      }
    }
    const s = decoder.decode();
    for (const ch of s) {
      if (ch === "\r") continue;
      producedText += ch;
      if (ch === "\n") {
        currentLen = 0;
      } else {
        currentLen++;
        if (wrapWidth > 0 && currentLen >= wrapWidth) {
          producedText += "\n";
          currentLen = 0;
        }
      }
    }
  } finally {
    await handle.close();
  }
  return {
    text: producedText,
    originalSize: totalSize,
    originalSha256Prefix: sha.digest("hex").substring(0, 16)
  };
}
var MAX_FILE_SIZE = 50 * 1024 * 1024;
var READ_TIMEOUT_MS = 5e3;
async function isProblematicFile(filePath) {
  try {
    const stats = await fs.stat(filePath);
    if (stats.size === 0) return false;
    if (path.basename(filePath).startsWith(".")) return true;
    if (stats.size > MAX_FILE_SIZE) return true;
    const ext = path.extname(filePath);
    const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
    if (isKnownType && stats.size > 1024 * 1024) return true;
    const handle = await fs.open(filePath, "r");
    try {
      const buffer = new Uint8Array(32 * 1024);
      const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
      const sample = buffer.subarray(0, bytesRead);
      if (ContentTypeInterpreter.isUnstructuredBinary(sample)) return true;
      if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType)) return true;
    } finally {
      await handle.close();
    }
    return false;
  } catch {
    return true;
  }
}
async function readFileSafely(filePath, options = {}) {
  const stats = await fs.stat(filePath);
  if (stats.size > MAX_FILE_SIZE) throw new Error(`File too large: ${stats.size}`);
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
  try {
    const handle = await fs.open(filePath, "r");
    try {
      const buffer = new Uint8Array(stats.size);
      await handle.read(buffer, 0, stats.size, 0);
      return buffer;
    } finally {
      await handle.close();
    }
  } catch (e) {
    if (e.name === "AbortError") throw new Error(`Read timeout for ${filePath}`);
    throw e;
  } finally {
    clearTimeout(timeout);
  }
}
async function listFiles(dirPath, recursive = false) {
  let files = [];
  try {
    const entries = await fs.readdir(dirPath, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = path.join(dirPath, entry.name);
      if (entry.name.startsWith(".")) continue;
      if (entry.isDirectory()) {
        if (recursive) {
          files = files.concat(await listFiles(fullPath, true));
        }
      } else if (entry.isFile()) {
        if (!await isProblematicFile(fullPath)) {
          files.push(fullPath);
        }
      }
    }
  } catch (e) {
    console.warn(`Error listing directory ${dirPath}:`, e);
  }
  return files;
}
async function processFileContent(filePath, options = {}) {
  const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
  const sample = rawContent.subarray(0, 1024 * 1024);
  const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
  let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
  if (options.forceBinary) isBinary = true;
  let content = rawContent;
  if (!isBinary) {
    try {
      content = new TextDecoder("utf-8", { fatal: true }).decode(rawContent);
    } catch {
      content = new TextDecoder("utf-8", { fatal: false }).decode(rawContent);
    }
  }
  return {
    content,
    filename: path.basename(filePath),
    mimeType: detection.mimeType,
    extension: detection.extension,
    isBinary,
    size: rawContent.length
  };
}

// src/Loader.ts
var DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024;
var WRAP_WIDTH_KNOWN = 1e3;
var WRAP_WIDTH_DEFAULT = 80;
async function processAndStoreFile(filePath, collection, options = {}) {
  const {
    allowProblematic = false,
    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
    metadataOnly = false,
    rootPath
  } = options;
  try {
    let fileInfo;
    if (await isProblematicFile(filePath)) {
      if (!allowProblematic) {
        console.warn(`Skipping problematic file: ${filePath}`);
        return null;
      }
      const extension = path2.extname(filePath).toLowerCase();
      const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
      const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
      console.warn(`Problematic file detected, processing as safe text: ${filePath}`);
      try {
        const streamed = await streamReadNormalizedText(filePath, {
          byteCap: maxBytesOnProblem,
          wrapWidth
        });
        fileInfo = {
          content: streamed.text,
          filename: path2.basename(filePath),
          mimeType: "text/plain",
          extension,
          isBinary: false,
          size: streamed.text.length,
          originalSize: streamed.originalSize,
          originalSha256Prefix: streamed.originalSha256Prefix,
          normalized: true,
          wrapWidth
        };
      } catch (e) {
        console.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`);
        fileInfo = await processFileContent(filePath, {
          forceBinary: true,
          allowPathological: true,
          maxBytes: maxBytesOnProblem
        });
      }
    } else {
      console.log(`Processing file: ${filePath}`);
      fileInfo = await processFileContent(filePath);
    }
    if (!fileInfo) return null;
    const content = fileInfo.content;
    if (!content || typeof content === "string" && content.length === 0 || content instanceof Uint8Array && content.length === 0) {
      if (process.env.DEBUG) {
        console.log(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
      }
      return {
        hash: "",
        contentType: fileInfo.mimeType,
        isBinary: fileInfo.isBinary,
        filename: fileInfo.filename,
        size: 0,
        filePath
      };
    }
    let mcard = null;
    const isProblematic = await isProblematicFile(filePath);
    if (metadataOnly && isProblematic) {
      mcard = null;
    } else {
      mcard = await MCard.create(fileInfo.content);
      const handle = path2.basename(filePath);
      try {
        await collection.addWithHandle(mcard, handle);
      } catch (e) {
        let registered = false;
        if (rootPath) {
          const relPath = path2.relative(rootPath, filePath);
          if (relPath !== handle) {
            try {
              await collection.addWithHandle(mcard, relPath);
              registered = true;
            } catch (e2) {
              if (process.env.DEBUG) {
                console.log(
                  `Handle name '${handle}' already in use (common for files like README.md, LICENSE). MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`
                );
              }
            }
          }
        }
        if (!registered) {
          try {
            await collection.add(mcard);
          } catch (e3) {
          }
        }
      }
    }
    const result = {
      hash: mcard ? mcard.hash : "METADATA_ONLY",
      contentType: fileInfo.mimeType,
      isBinary: fileInfo.isBinary,
      filename: fileInfo.filename,
      size: fileInfo.size,
      filePath
    };
    if (fileInfo.originalSize !== void 0) result.originalSize = fileInfo.originalSize;
    if (fileInfo.originalSha256Prefix) result.originalSha256Prefix = fileInfo.originalSha256Prefix;
    if (metadataOnly && isProblematic) result.metadataOnly = true;
    return result;
  } catch (e) {
    console.error(`Error processing ${filePath}:`, e);
    return null;
  }
}
async function loadFileToCollection(targetPath, collection, options = {}) {
  const {
    recursive = false,
    includeProblematic = false,
    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
    metadataOnly = false
  } = options;
  const resolvedPath = path2.resolve(targetPath);
  const stats = await fs2.stat(resolvedPath);
  const results = [];
  let files = [];
  let rootPath = resolvedPath;
  if (stats.isFile()) {
    files = [resolvedPath];
    rootPath = path2.dirname(resolvedPath);
  } else if (stats.isDirectory()) {
    files = await listFiles(resolvedPath, recursive);
    rootPath = resolvedPath;
  } else {
    throw new Error(`Path ${targetPath} is not a file or directory`);
  }
  const uniqueDirs = /* @__PURE__ */ new Set();
  let maxDepth = 0;
  for (const file of files) {
    const dir = path2.dirname(file);
    if (dir.startsWith(rootPath)) {
      uniqueDirs.add(dir);
      const rel = path2.relative(rootPath, file);
      const parts = rel.split(path2.sep);
      const depth = parts.length - 1;
      if (depth > maxDepth) maxDepth = depth;
    }
  }
  const metrics = {
    filesCount: files.length,
    directoriesCount: uniqueDirs.size,
    directoryLevels: maxDepth
  };
  console.log(`About to process ${files.length} files`);
  for (const file of files) {
    const result = await processAndStoreFile(file, collection, {
      allowProblematic: includeProblematic,
      maxBytesOnProblem,
      metadataOnly,
      rootPath
    });
    if (result) results.push(result);
  }
  return { metrics, results };
}

export {
  FileIO_exports,
  processAndStoreFile,
  loadFileToCollection,
  Loader_exports
};