@2501-ai/cli
Version:
[](https://www.npmjs.com/package/@2501-ai/cli) [](https://www.2501.ai/research/full-humaneval-benchmark) [![Lic
131 lines (130 loc) • 5.21 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.computeFileMetadataHash = exports.getDirectoryMd5Hash = exports.getDirectoryFiles = exports.toReadableSize = exports.isTextFile = void 0;
const crypto_1 = __importDefault(require("crypto"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const logger_1 = __importDefault(require("../utils/logger"));
const constants_1 = require("../constants");
const ignore_1 = require("./ignore");
const istextorbinary_1 = require("istextorbinary");
function isTextFile(filePath) {
const extension = path_1.default.extname(filePath).toLowerCase();
if (constants_1.INCLUDED_FILE_EXTENSIONS.includes(extension)) {
return true;
}
return (0, istextorbinary_1.isText)(filePath);
}
exports.isTextFile = isTextFile;
const toReadableSize = (bytes) => {
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
if (bytes === 0)
return '0 Bytes';
const i = Math.floor(Math.log(bytes) / Math.log(1024));
const size = (bytes / Math.pow(1024, i)).toFixed(2);
return `${size} ${sizes[i]}`;
};
exports.toReadableSize = toReadableSize;
function hasReachedThreshold(totalSize, maxSize) {
return totalSize / maxSize >= 0.999;
}
function getDirectoryFiles(params) {
if (params.currentDepth > params.maxDepth) {
return {
totalSize: 0,
fileHashes: new Map(),
};
}
const gitignorePath = path_1.default.join(params.directoryPath, params.currentPath, '.gitignore');
if (fs_1.default.existsSync(gitignorePath)) {
params.ignoreManager.loadGitignore(gitignorePath, params.currentPath);
}
const fileHashes = new Map();
let items = [];
try {
items = fs_1.default.readdirSync(path_1.default.join(params.directoryPath, params.currentPath), { withFileTypes: true });
}
catch (e) {
logger_1.default.error(`Error reading directory: ${e.message}`, e);
return {
totalSize: 0,
fileHashes: new Map(),
};
}
let totalSize = 0;
for (const item of items) {
const itemRelativePath = path_1.default.join(params.currentPath, item.name);
const itemFullPath = path_1.default.join(params.directoryPath, itemRelativePath);
if (params.ignoreManager.isIgnored(itemRelativePath)) {
continue;
}
if (item.isDirectory()) {
const result = getDirectoryFiles(Object.assign(Object.assign({}, params), { currentPath: itemRelativePath, currentDepth: params.currentDepth + 1, currentTotalSize: totalSize + params.currentTotalSize }));
totalSize += result.totalSize;
result.fileHashes.forEach((hash, relativePath) => {
fileHashes.set(relativePath, hash);
});
}
else if (item.isFile()) {
const { hash: fileHash, size: fileSize } = computeFileMetadataHash(itemFullPath);
if (fileSize > constants_1.DEFAULT_MAX_FILE_SIZE) {
continue;
}
const sizeWithFile = totalSize + params.currentTotalSize + fileSize;
if (sizeWithFile >= params.maxDirSize) {
if (hasReachedThreshold(totalSize + params.currentTotalSize, params.maxDirSize)) {
break;
}
continue;
}
fileHashes.set(itemRelativePath, fileHash);
totalSize += fileSize;
}
}
return {
totalSize,
fileHashes,
};
}
exports.getDirectoryFiles = getDirectoryFiles;
function getDirectoryMd5Hash({ directoryPath, maxDepth = constants_1.DEFAULT_MAX_DEPTH, maxDirSize = constants_1.DEFAULT_MAX_DIR_SIZE, }) {
const ignoreManager = ignore_1.IgnoreManager.getInstance();
const result = getDirectoryFiles({
directoryPath,
maxDepth,
maxDirSize,
currentPath: '',
currentDepth: 0,
currentTotalSize: 0,
ignoreManager,
});
logger_1.default.debug('Total size of files in directory:', (0, exports.toReadableSize)(result.totalSize));
logger_1.default.debug('Total files hashed:', result.fileHashes.size);
const sortedFileHashes = Array.from(result.fileHashes)
.sort(([pathA], [pathB]) => pathA.localeCompare(pathB))
.map(([relativePath, fileHash]) => `${fileHash}:${relativePath}`);
const md5 = crypto_1.default
.createHash('md5')
.update(sortedFileHashes.join('|'))
.digest('hex');
return {
md5,
fileHashes: result.fileHashes,
directoryPath,
totalSize: result.totalSize,
};
}
exports.getDirectoryMd5Hash = getDirectoryMd5Hash;
function computeFileMetadataHash(filePath) {
const stats = fs_1.default.statSync(filePath);
const metaHash = crypto_1.default.createHash('md5');
metaHash.update(`${stats.size}:${stats.mtimeMs}`);
return {
hash: metaHash.digest('hex'),
size: stats.size,
};
}
exports.computeFileMetadataHash = computeFileMetadataHash;