@huggingface/dduf
Version:
Very alpha lib to check DDUF compliance
161 lines (157 loc) • 7.87 kB
JavaScript
;Object.defineProperty(exports, "__esModule", {value: true});// src/check-filename.ts
function checkFilename(filename) {
if (!filename.endsWith(".safetensors") && !filename.endsWith(".json") && !filename.endsWith(".gguf") && !filename.endsWith(".txt") && !filename.endsWith("/")) {
throw new Error("Files must have a .safetensors, .txt, .gguf or .json extension");
}
const split = filename.split("/");
if (split.length > 2) {
throw new Error("Files must be only one level deep, not more");
}
}
// src/check-dduf.ts
var _blob = require('@huggingface/blob');
async function* checkDDUF(url, opts) {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o;
const blob = url instanceof Blob ? url : await _blob.createBlob.call(void 0, url);
(_a = opts == null ? void 0 : opts.log) == null ? void 0 : _a.call(opts, "File size: " + blob.size);
const last100kB = await blob.slice(blob.size - 1e5, blob.size).arrayBuffer();
const view = new DataView(last100kB);
let index = view.byteLength - 22;
let found = false;
while (index >= 0) {
if (view.getUint32(index, true) === 101010256) {
found = true;
break;
}
index--;
}
if (!found) {
throw new Error("DDUF footer not found in last 100kB of file");
}
(_b = opts == null ? void 0 : opts.log) == null ? void 0 : _b.call(opts, "DDUF footer found at offset " + (blob.size - last100kB.byteLength + index));
const diskNumber = view.getUint16(index + 4, true);
(_c = opts == null ? void 0 : opts.log) == null ? void 0 : _c.call(opts, "Disk number: " + diskNumber);
if (diskNumber !== 0 && diskNumber !== 65535) {
throw new Error("Multi-disk archives not supported");
}
let fileCount = view.getUint16(index + 10, true);
let centralDirSize = view.getUint32(index + 12, true);
let centralDirOffset = view.getUint32(index + 16, true);
const isZip64 = centralDirOffset === 4294967295;
(_d = opts == null ? void 0 : opts.log) == null ? void 0 : _d.call(opts, "File count: " + fileCount);
if (isZip64) {
(_e = opts == null ? void 0 : opts.log) == null ? void 0 : _e.call(opts, "Zip64 format detected");
index -= 20;
found = false;
while (index >= 0) {
if (view.getUint32(index, true) === 117853008) {
found = true;
break;
}
index--;
}
if (!found) {
throw new Error("Zip64 footer not found in last 100kB of file");
}
(_f = opts == null ? void 0 : opts.log) == null ? void 0 : _f.call(opts, "Zip64 footer found at offset " + (blob.size - last100kB.byteLength + index));
const diskWithCentralDir = view.getUint32(index + 4, true);
if (diskWithCentralDir !== 0) {
throw new Error("Multi-disk archives not supported");
}
const endCentralDirOffset = Number(view.getBigUint64(index + 8, true));
index = endCentralDirOffset - (blob.size - last100kB.byteLength);
if (index < 0) {
throw new Error("Central directory offset is outside the last 100kB of the file");
}
if (view.getUint32(index, true) !== 101075792) {
throw new Error("Invalid central directory header");
}
const thisDisk = view.getUint16(index + 16, true);
const centralDirDisk = view.getUint16(index + 20, true);
if (thisDisk !== 0) {
throw new Error("Multi-disk archives not supported");
}
if (centralDirDisk !== 0) {
throw new Error("Multi-disk archives not supported");
}
centralDirSize = Number(view.getBigUint64(index + 40, true));
centralDirOffset = Number(view.getBigUint64(index + 48, true));
fileCount = Number(view.getBigUint64(index + 32, true));
(_g = opts == null ? void 0 : opts.log) == null ? void 0 : _g.call(opts, "File count zip 64: " + fileCount);
}
(_h = opts == null ? void 0 : opts.log) == null ? void 0 : _h.call(opts, "Central directory size: " + centralDirSize);
(_i = opts == null ? void 0 : opts.log) == null ? void 0 : _i.call(opts, "Central directory offset: " + centralDirOffset);
const centralDir = centralDirOffset > blob.size - last100kB.byteLength ? last100kB.slice(
centralDirOffset - (blob.size - last100kB.byteLength),
centralDirOffset - (blob.size - last100kB.byteLength) + centralDirSize
) : await blob.slice(centralDirOffset, centralDirOffset + centralDirSize).arrayBuffer();
const centralDirView = new DataView(centralDir);
let offset = 0;
for (let i = 0; i < fileCount; i++) {
if (centralDirView.getUint32(offset + 0, true) !== 33639248) {
throw new Error("Invalid central directory file header");
}
if (offset + 46 > centralDir.byteLength) {
throw new Error("Unexpected end of central directory");
}
const compressionMethod = centralDirView.getUint16(offset + 10, true);
if (compressionMethod !== 0) {
throw new Error("Unsupported compression method: " + compressionMethod);
}
const filenameLength = centralDirView.getUint16(offset + 28, true);
const fileName = new TextDecoder().decode(new Uint8Array(centralDir, offset + 46, filenameLength));
(_j = opts == null ? void 0 : opts.log) == null ? void 0 : _j.call(opts, "File " + i);
(_k = opts == null ? void 0 : opts.log) == null ? void 0 : _k.call(opts, "File name: " + fileName);
checkFilename(fileName);
const fileDiskNumber = centralDirView.getUint16(34, true);
if (fileDiskNumber !== 0 && fileDiskNumber !== 65535) {
throw new Error("Multi-disk archives not supported");
}
let size = centralDirView.getUint32(offset + 24, true);
let compressedSize = centralDirView.getUint32(offset + 20, true);
let filePosition = centralDirView.getUint32(offset + 42, true);
const extraFieldLength = centralDirView.getUint16(offset + 30, true);
if (size === 4294967295 || compressedSize === 4294967295 || filePosition === 4294967295) {
(_l = opts == null ? void 0 : opts.log) == null ? void 0 : _l.call(opts, "File size is in zip64 format");
const extraFields = new DataView(centralDir, offset + 46 + filenameLength, extraFieldLength);
let extraFieldOffset = 0;
while (extraFieldOffset < extraFieldLength) {
const headerId = extraFields.getUint16(extraFieldOffset, true);
const extraFieldSize = extraFields.getUint16(extraFieldOffset + 2, true);
if (headerId !== 1) {
extraFieldOffset += 4 + extraFieldSize;
continue;
}
const zip64ExtraField = new DataView(
centralDir,
offset + 46 + filenameLength + extraFieldOffset + 4,
extraFieldSize
);
let zip64ExtraFieldOffset = 0;
if (size === 4294967295) {
size = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
if (compressedSize === 4294967295) {
compressedSize = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
if (filePosition === 4294967295) {
filePosition = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
break;
}
}
if (size !== compressedSize) {
throw new Error("Compressed size and size differ: " + compressedSize + " vs " + size);
}
(_m = opts == null ? void 0 : opts.log) == null ? void 0 : _m.call(opts, "File size: " + size);
const commentLength = centralDirView.getUint16(offset + 32, true);
(_n = opts == null ? void 0 : opts.log) == null ? void 0 : _n.call(opts, "File header position in archive: " + filePosition);
offset += 46 + filenameLength + extraFieldLength + commentLength;
yield { type: "file", name: fileName, size, fileHeaderOffset: filePosition };
}
(_o = opts == null ? void 0 : opts.log) == null ? void 0 : _o.call(opts, "All files checked");
}
exports.checkDDUF = checkDDUF;