@huggingface/dduf
Version:
Very alpha lib to check DDUF compliance
187 lines (183 loc) • 7.83 kB
JavaScript
;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// index.ts
var dduf_exports = {};
__export(dduf_exports, {
checkDDUF: () => checkDDUF
});
module.exports = __toCommonJS(dduf_exports);
// src/check-filename.ts
function checkFilename(filename) {
if (!filename.endsWith(".safetensors") && !filename.endsWith(".json") && !filename.endsWith(".gguf") && !filename.endsWith(".txt") && !filename.endsWith("/")) {
throw new Error("Files must have a .safetensors, .txt, .gguf or .json extension");
}
const split = filename.split("/");
if (split.length > 2) {
throw new Error("Files must be only one level deep, not more");
}
}
// src/check-dduf.ts
var import_blob = require("@huggingface/blob");
async function* checkDDUF(url, opts) {
const blob = url instanceof Blob ? url : await (0, import_blob.createBlob)(url);
opts?.log?.("File size: " + blob.size);
const last100kB = await blob.slice(blob.size - 1e5, blob.size).arrayBuffer();
const view = new DataView(last100kB);
let index = view.byteLength - 22;
let found = false;
while (index >= 0) {
if (view.getUint32(index, true) === 101010256) {
found = true;
break;
}
index--;
}
if (!found) {
throw new Error("DDUF footer not found in last 100kB of file");
}
opts?.log?.("DDUF footer found at offset " + (blob.size - last100kB.byteLength + index));
const diskNumber = view.getUint16(index + 4, true);
opts?.log?.("Disk number: " + diskNumber);
if (diskNumber !== 0 && diskNumber !== 65535) {
throw new Error("Multi-disk archives not supported");
}
let fileCount = view.getUint16(index + 10, true);
let centralDirSize = view.getUint32(index + 12, true);
let centralDirOffset = view.getUint32(index + 16, true);
const isZip64 = centralDirOffset === 4294967295;
opts?.log?.("File count: " + fileCount);
if (isZip64) {
opts?.log?.("Zip64 format detected");
index -= 20;
found = false;
while (index >= 0) {
if (view.getUint32(index, true) === 117853008) {
found = true;
break;
}
index--;
}
if (!found) {
throw new Error("Zip64 footer not found in last 100kB of file");
}
opts?.log?.("Zip64 footer found at offset " + (blob.size - last100kB.byteLength + index));
const diskWithCentralDir = view.getUint32(index + 4, true);
if (diskWithCentralDir !== 0) {
throw new Error("Multi-disk archives not supported");
}
const endCentralDirOffset = Number(view.getBigUint64(index + 8, true));
index = endCentralDirOffset - (blob.size - last100kB.byteLength);
if (index < 0) {
throw new Error("Central directory offset is outside the last 100kB of the file");
}
if (view.getUint32(index, true) !== 101075792) {
throw new Error("Invalid central directory header");
}
const thisDisk = view.getUint16(index + 16, true);
const centralDirDisk = view.getUint16(index + 20, true);
if (thisDisk !== 0) {
throw new Error("Multi-disk archives not supported");
}
if (centralDirDisk !== 0) {
throw new Error("Multi-disk archives not supported");
}
centralDirSize = Number(view.getBigUint64(index + 40, true));
centralDirOffset = Number(view.getBigUint64(index + 48, true));
fileCount = Number(view.getBigUint64(index + 32, true));
opts?.log?.("File count zip 64: " + fileCount);
}
opts?.log?.("Central directory size: " + centralDirSize);
opts?.log?.("Central directory offset: " + centralDirOffset);
const centralDir = centralDirOffset > blob.size - last100kB.byteLength ? last100kB.slice(
centralDirOffset - (blob.size - last100kB.byteLength),
centralDirOffset - (blob.size - last100kB.byteLength) + centralDirSize
) : await blob.slice(centralDirOffset, centralDirOffset + centralDirSize).arrayBuffer();
const centralDirView = new DataView(centralDir);
let offset = 0;
for (let i = 0; i < fileCount; i++) {
if (centralDirView.getUint32(offset + 0, true) !== 33639248) {
throw new Error("Invalid central directory file header");
}
if (offset + 46 > centralDir.byteLength) {
throw new Error("Unexpected end of central directory");
}
const compressionMethod = centralDirView.getUint16(offset + 10, true);
if (compressionMethod !== 0) {
throw new Error("Unsupported compression method: " + compressionMethod);
}
const filenameLength = centralDirView.getUint16(offset + 28, true);
const fileName = new TextDecoder().decode(new Uint8Array(centralDir, offset + 46, filenameLength));
opts?.log?.("File " + i);
opts?.log?.("File name: " + fileName);
checkFilename(fileName);
const fileDiskNumber = centralDirView.getUint16(34, true);
if (fileDiskNumber !== 0 && fileDiskNumber !== 65535) {
throw new Error("Multi-disk archives not supported");
}
let size = centralDirView.getUint32(offset + 24, true);
let compressedSize = centralDirView.getUint32(offset + 20, true);
let filePosition = centralDirView.getUint32(offset + 42, true);
const extraFieldLength = centralDirView.getUint16(offset + 30, true);
if (size === 4294967295 || compressedSize === 4294967295 || filePosition === 4294967295) {
opts?.log?.("File size is in zip64 format");
const extraFields = new DataView(centralDir, offset + 46 + filenameLength, extraFieldLength);
let extraFieldOffset = 0;
while (extraFieldOffset < extraFieldLength) {
const headerId = extraFields.getUint16(extraFieldOffset, true);
const extraFieldSize = extraFields.getUint16(extraFieldOffset + 2, true);
if (headerId !== 1) {
extraFieldOffset += 4 + extraFieldSize;
continue;
}
const zip64ExtraField = new DataView(
centralDir,
offset + 46 + filenameLength + extraFieldOffset + 4,
extraFieldSize
);
let zip64ExtraFieldOffset = 0;
if (size === 4294967295) {
size = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
if (compressedSize === 4294967295) {
compressedSize = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
if (filePosition === 4294967295) {
filePosition = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true));
zip64ExtraFieldOffset += 8;
}
break;
}
}
if (size !== compressedSize) {
throw new Error("Compressed size and size differ: " + compressedSize + " vs " + size);
}
opts?.log?.("File size: " + size);
const commentLength = centralDirView.getUint16(offset + 32, true);
opts?.log?.("File header position in archive: " + filePosition);
offset += 46 + filenameLength + extraFieldLength + commentLength;
yield { type: "file", name: fileName, size, fileHeaderOffset: filePosition };
}
opts?.log?.("All files checked");
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
checkDDUF
});