@intlayer/chokidar
Version:
Uses chokidar to scan and build Intlayer declaration files into dictionaries based on Intlayer configuration.
337 lines (335 loc) • 11.4 kB
JavaScript
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
const require_utils_getChunk = require('./getChunk.cjs');
//#region src/utils/chunkJSON.ts
/**
* Split & reassemble JSON by character budget.
* - Measures serialized size using JSON.stringify(..).length (characters).
* - Ensures each chunk is itself valid JSON.
* - Very large strings are split into safe pieces using getChunk and re-concatenated on assemble.
* - Protects against circular structures (JSON can't serialize those anyway).
*/
const isObject = (val) => {
return typeof val === "object" && val !== null && !Array.isArray(val);
};
const computeDjb2 = (str) => {
let hash = 5381;
for (let i = 0; i < str.length; i++) hash = (hash << 5) + hash ^ str.charCodeAt(i);
return (hash >>> 0).toString(16).padStart(8, "0");
};
const setAtPath = (root, path, value) => {
let current = root;
for (let i = 0; i < path.length - 1; i++) {
const key = path[i];
const isNextIndex = typeof path[i + 1] === "number";
if (typeof key === "number") {
if (!Array.isArray(current)) throw new Error(`Expected array at path segment ${i}`);
if (current[key] === void 0) current[key] = isNextIndex ? [] : {};
current = current[key];
} else {
if (!isObject(current)) throw new Error(`Expected object at path segment ${i}`);
if (!(key in current)) current[key] = isNextIndex ? [] : {};
current = current[key];
}
}
const last = path[path.length - 1];
if (typeof last === "number") {
if (!Array.isArray(current)) throw new Error(`Expected array at final segment`);
current[last] = value;
} else {
if (!isObject(current)) throw new Error(`Expected object at final segment`);
current[last] = value;
}
};
const pathKey = (path) => {
return JSON.stringify(path);
};
/**
* Split a string into parts using getChunk with a charLength budget per part.
*/
const splitStringByBudget = (str, maxCharsPerPart) => {
if (maxCharsPerPart <= 0) throw new Error("maxChars must be > 0");
const output = [];
let offset = 0;
while (offset < str.length) {
const part = require_utils_getChunk.getChunk(str, {
charStart: offset,
charLength: maxCharsPerPart
});
if (!part) break;
output.push(part);
offset += part.length;
}
return output;
};
/**
* Flatten JSON into patches (leaf writes). Strings too large to fit in a single
* chunk are yielded as multiple str-append patches.
*/
const flattenToPatches = (value, maxCharsPerChunk, path = [], seen = /* @__PURE__ */ new WeakSet()) => {
const maxStringPiece = Math.max(1, Math.floor((maxCharsPerChunk - 400) * .8));
const patches = [];
const walk = (currentValue, currentPath) => {
if (typeof currentValue === "string") {
const testPatch = {
op: "set",
path: currentPath,
value: currentValue
};
if (JSON.stringify(testPatch).length + 150 <= maxCharsPerChunk) {
patches.push(testPatch);
return;
}
const parts = splitStringByBudget(currentValue, maxStringPiece);
patches.push({
op: "set",
path: [...currentPath, "__splittedType"],
value: "string"
});
patches.push({
op: "set",
path: [...currentPath, "__total"],
value: parts.length
});
for (let i = 0; i < parts.length; i++) patches.push({
op: "set",
path: [...currentPath, String(i + 1)],
value: parts[i]
});
return;
}
if (currentValue === null || typeof currentValue !== "object") {
patches.push({
op: "set",
path: currentPath,
value: currentValue
});
return;
}
if (seen.has(currentValue)) throw new Error("Cannot serialize circular structures to JSON.");
seen.add(currentValue);
if (Array.isArray(currentValue)) for (let i = 0; i < currentValue.length; i++) walk(currentValue[i], [...currentPath, i]);
else for (const key of Object.keys(currentValue)) walk(currentValue[key], [...currentPath, key]);
seen.delete(currentValue);
};
walk(value, path);
return patches;
};
/**
* Split JSON into chunks constrained by character count of serialized chunk.
*/
const chunkJSON = (value, maxChars) => {
if (!isObject(value) && !Array.isArray(value)) throw new Error("Root must be an object or array.");
if (maxChars < 500) throw new Error("maxChars is too small. Use at least 500 characters.");
const rootType = Array.isArray(value) ? "array" : "object";
let sourceString;
try {
sourceString = JSON.stringify(value);
} catch {
throw new Error("Cannot serialize circular structures to JSON.");
}
const checksum = computeDjb2(sourceString);
const allPatches = flattenToPatches(value, maxChars);
const chunks = [];
let currentChunk = {
schemaVersion: 1,
index: 0,
total: 0,
rootType,
checksum,
entries: []
};
const emptyEnvelopeSize = JSON.stringify({
...currentChunk,
entries: []
}).length;
const tryFlush = () => {
if (currentChunk.entries.length > 0) {
chunks.push(currentChunk);
currentChunk = {
schemaVersion: 1,
index: 0,
total: 0,
rootType,
checksum,
entries: []
};
}
};
for (const patch of allPatches) if (emptyEnvelopeSize + JSON.stringify(currentChunk.entries).length + (currentChunk.entries.length ? 1 : 0) + JSON.stringify(patch).length <= maxChars) currentChunk.entries.push(patch);
else {
if (currentChunk.entries.length > 0) tryFlush();
if (emptyEnvelopeSize + JSON.stringify([patch]).length > maxChars) throw new Error("A single entry exceeds maxChars and cannot be split. Reduce entry size or increase maxChars.");
currentChunk.entries.push(patch);
}
tryFlush();
if (chunks.length === 0) chunks.push({
schemaVersion: 1,
index: 0,
total: 0,
rootType,
checksum,
entries: []
});
const totalChunks = chunks.length;
chunks.forEach((chunk, index) => {
chunk.index = index;
chunk.total = totalChunks;
});
return chunks;
};
/**
* Reassemble JSON from chunks.
* - Validates checksums and indices.
* - Applies 'set' patches and merges string pieces from 'str-append'.
*/
/**
* Reconstruct content from a single chunk without validation.
* Useful for processing individual chunks in a pipeline where you don't have all chunks yet.
* Note: This will only reconstruct the partial content contained in this chunk.
*/
const reconstructFromSingleChunk = (chunk) => {
const root = chunk.rootType === "array" ? [] : {};
for (const entry of chunk.entries) if (entry.op === "set") setAtPath(root, entry.path, entry.value);
const reconcileSplitNodes = (node) => {
if (node === null || typeof node !== "object") return node;
if (Array.isArray(node)) {
for (let i = 0; i < node.length; i++) node[i] = reconcileSplitNodes(node[i]);
return node;
}
if (node["__splittedType"] === "string") {
const total = node["__total"];
if (typeof total !== "number" || total <= 0) return node;
const parts = [];
let hasAllParts = true;
for (let i = 1; i <= total; i++) {
const piece = node[String(i)];
if (typeof piece !== "string") {
hasAllParts = false;
break;
}
parts.push(piece);
}
if (hasAllParts) return parts.join("");
return node;
}
if (node["__splittedType"] === "array") {
const total = node["__total"];
if (typeof total !== "number" || total < 0) return node;
const output = [];
let hasAllParts = true;
for (let i = 1; i <= total; i++) {
const slice = node[String(i)];
if (!Array.isArray(slice)) {
hasAllParts = false;
break;
}
for (let j = 0; j < slice.length; j++) output.push(reconcileSplitNodes(slice[j]));
}
if (hasAllParts) return output;
return node;
}
for (const key of Object.keys(node)) node[key] = reconcileSplitNodes(node[key]);
return node;
};
return reconcileSplitNodes(root);
};
const assembleJSON = (chunks) => {
if (!chunks || chunks.length === 0) throw new Error("No chunks provided.");
const sorted = [...chunks].sort((a, b) => a.index - b.index);
const { checksum, rootType } = sorted[0];
const schemaVersion = 1;
for (let i = 0; i < sorted.length; i++) {
const chunk = sorted[i];
if (chunk.schemaVersion !== schemaVersion) {
console.error("Unsupported schemaVersion.", {
cause: chunk,
schemaVersion
});
throw new Error("Unsupported schemaVersion.");
}
if (chunk.rootType !== rootType) {
console.error("Chunks rootType mismatch.", {
cause: chunk,
rootType
});
throw new Error("Chunks rootType mismatch.");
}
if (chunk.checksum !== checksum) {
console.error("Chunks checksum mismatch (different source objects?).", {
cause: chunk,
checksum
});
throw new Error("Chunks checksum mismatch (different source objects?).");
}
if (chunk.index !== i) {
console.error("Chunk indices are not contiguous or sorted.", {
cause: chunk,
index: chunk.index,
i
});
throw new Error("Chunk indices are not contiguous or sorted.");
}
}
const root = rootType === "array" ? [] : {};
const stringParts = /* @__PURE__ */ new Map();
const applySet = (patch) => setAtPath(root, patch.path, patch.value);
for (const chunk of sorted) for (const entry of chunk.entries) if (entry.op === "set") applySet(entry);
else {
const key = pathKey(entry.path);
const record = stringParts.get(key) ?? {
path: entry.path,
total: entry.total,
received: []
};
if (record.total !== entry.total) throw new Error("Inconsistent string part totals for a path.");
record.received.push(entry);
stringParts.set(key, record);
}
for (const { path, total, received } of stringParts.values()) {
if (received.length !== total) throw new Error("Missing string parts for a path; incomplete chunk set.");
received.sort((a, b) => a.index - b.index);
setAtPath(root, path, received.map((part) => part.value).join(""));
}
const reconcileSplitNodes = (node) => {
if (node === null || typeof node !== "object") return node;
if (Array.isArray(node)) {
for (let i = 0; i < node.length; i++) node[i] = reconcileSplitNodes(node[i]);
return node;
}
if (node["__splittedType"] === "string") {
const total = node["__total"];
if (typeof total !== "number" || total <= 0) throw new Error("Invalid split-node total for a path.");
const parts = [];
for (let i = 1; i <= total; i++) {
const piece = node[String(i)];
if (typeof piece !== "string") throw new Error("Missing string parts for a path; incomplete chunk set.");
parts.push(piece);
}
return parts.join("");
}
if (node["__splittedType"] === "array") {
const total = node["__total"];
if (typeof total !== "number" || total < 0) throw new Error("Invalid split-node total for a path.");
const output = [];
for (let i = 1; i <= total; i++) {
const slice = node[String(i)];
if (!Array.isArray(slice)) throw new Error("Missing string parts for a path; incomplete chunk set.");
for (let j = 0; j < slice.length; j++) output.push(reconcileSplitNodes(slice[j]));
}
return output;
}
for (const key of Object.keys(node)) node[key] = reconcileSplitNodes(node[key]);
return node;
};
const reconciled = reconcileSplitNodes(root);
for (let i = 0; i < sorted.length; i++) {
const chunk = sorted[i];
if (chunk.total !== sorted.length) throw new Error(`Chunk total does not match provided count. Expected ${sorted.length}, but chunk ${i} has total=${chunk.total}`);
}
return reconciled;
};
//#endregion
exports.assembleJSON = assembleJSON;
exports.chunkJSON = chunkJSON;
exports.reconstructFromSingleChunk = reconstructFromSingleChunk;
//# sourceMappingURL=chunkJSON.cjs.map