@jackhua/mini-langchain
Version:
A lightweight TypeScript implementation of LangChain with cost optimization features
99 lines • 3.25 kB
JavaScript
;
/**
* Directory document loader
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.DirectoryLoader = void 0;
exports.createLoader = createLoader;
const promises_1 = require("fs/promises");
const path_1 = require("path");
const base_1 = require("./base");
const text_1 = require("./text");
/**
* Load all documents from a directory
*/
class DirectoryLoader extends base_1.BaseDocumentLoader {
constructor(directoryPath, options = {}) {
super();
this.directoryPath = directoryPath;
this.glob = options.glob || '**/*';
this.loader = options.loader || ((path) => new text_1.TextLoader(path));
this.recursive = options.recursive ?? true;
this.silent = options.silent ?? false;
}
async load() {
const documents = [];
try {
const files = await this.getFiles(this.directoryPath);
for (const file of files) {
if (this.matchesGlob(file)) {
try {
const loader = this.loader(file);
const docs = await loader.load();
documents.push(...docs);
}
catch (error) {
if (!this.silent) {
console.warn(`Failed to load ${file}: ${error}`);
}
}
}
}
return documents;
}
catch (error) {
throw new Error(`Failed to load directory ${this.directoryPath}: ${error}`);
}
}
async getFiles(dir) {
const files = [];
const entries = await (0, promises_1.readdir)(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = (0, path_1.join)(dir, entry.name);
if (entry.isDirectory() && this.recursive) {
const subFiles = await this.getFiles(fullPath);
files.push(...subFiles);
}
else if (entry.isFile()) {
files.push(fullPath);
}
}
return files;
}
matchesGlob(filePath) {
// Simple glob matching (can be enhanced)
if (this.glob === '**/*')
return true;
// Match by extension
if (this.glob.startsWith('*.')) {
const ext = this.glob.substring(1);
return filePath.endsWith(ext);
}
// Match by pattern
if (this.glob.includes('*')) {
const pattern = this.glob
.replace(/\./g, '\\.')
.replace(/\*/g, '.*');
const regex = new RegExp(pattern);
return regex.test(filePath);
}
return filePath.includes(this.glob);
}
}
exports.DirectoryLoader = DirectoryLoader;
/**
* Create loader based on file extension
*/
function createLoader(filePath) {
const ext = filePath.split('.').pop()?.toLowerCase();
switch (ext) {
case 'txt':
case 'text':
case 'md':
case 'markdown':
return new text_1.TextLoader(filePath);
default:
return new text_1.TextLoader(filePath);
}
}
//# sourceMappingURL=directory.js.map