@hpbyte/h-codex-core
Version:
Core indexing and search functionality for h-codex
111 lines • 4.49 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.cstChunker = exports.CSTChunker = void 0;
const crypto_1 = __importDefault(require("crypto"));
const utils_1 = require("../../utils");
const tree_sitter_parser_1 = require("./tree-sitter-parser");
const chunk_range_1 = require("./chunk-range");
class CSTChunker {
constructor() {
this.filePath = '';
this.fileContent = '';
this.projectId = '';
this.language = null;
}
async chunk({ filePath, projectId, language }) {
const cstRootNode = await tree_sitter_parser_1.treeSitterParser.parseFile(filePath, language);
this.filePath = filePath;
this.language = language;
this.fileContent = cstRootNode.text;
this.projectId = projectId;
const chunks = this.chunkNode(cstRootNode);
const processedChunks = this.processChunks(chunks);
return processedChunks;
}
/**
* AST-based chunking
* - each nodes are traversed and greedily bundled together
* - if a node is too large, recursively chunk its children
* - else, concatenate the current chunk to the node
*/
chunkNode(node) {
const chunks = [];
let currentChunk = null;
const addCurrentChunk = () => {
if (currentChunk && !currentChunk.isEmpty) {
chunks.push(currentChunk);
currentChunk = null;
}
};
for (const child of node.children) {
const childChunkRange = new chunk_range_1.ChunkRange(child.startPosition.row, child.endPosition.row, child.type);
const childText = child.text;
if (childText.length > utils_1.maxChunkSize) {
addCurrentChunk();
chunks.push(...this.chunkNode(child));
continue;
}
const currentChunkText = currentChunk?.extract(this.fileContent) ?? '';
const wouldExceedSize = currentChunkText.length + childText.length > utils_1.maxChunkSize;
if (wouldExceedSize) {
addCurrentChunk();
currentChunk = childChunkRange;
}
else {
currentChunk = currentChunk ? currentChunk.add(childChunkRange) : childChunkRange;
}
}
addCurrentChunk();
return chunks;
}
/**
* try to combine very small chunks (under the coalesce threshold) with adjacent chunks
* but if combing will exceed the max chunk size, keep as is
*/
processChunks(chunks) {
if (chunks.length === 0)
return [];
const processedChunks = [];
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
const chunkText = chunk.extract(this.fileContent);
if ((0, utils_1.countLengthWithoutWhitespace)(chunkText) >= utils_1.coalesce) {
processedChunks.push(this.formatAsCodeChunk(chunk, chunkText));
continue;
}
if (i < chunks.length - 1) {
const nextChunk = chunks[i + 1];
const combinedChunk = chunk.add(nextChunk);
const combinedText = combinedChunk.extract(this.fileContent);
if (combinedText.length <= utils_1.maxChunkSize) {
processedChunks.push(this.formatAsCodeChunk(combinedChunk, combinedText));
// skip the next chunk as it's combined with the current one
i++;
continue;
}
}
// can't coalesce, keep as is
processedChunks.push(this.formatAsCodeChunk(chunk, chunkText));
}
return processedChunks;
}
formatAsCodeChunk(chunk, content) {
return {
content,
filePath: this.filePath,
projectId: this.projectId,
startLine: chunk.start + 1, // tree-sitter index starts from 0
endLine: chunk.end + 1,
nodeType: chunk.nodeType,
language: this.language,
hash: crypto_1.default.createHash('sha256').update(content).digest('hex'),
size: content.length,
};
}
}
exports.CSTChunker = CSTChunker;
exports.cstChunker = new CSTChunker();
//# sourceMappingURL=cst-chunker.js.map