@twilio/plugin-microvisor
Version:
Interact with your Twilio Microvisor devices
279 lines (278 loc) • 11.6 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LittleFsWriter = exports.ctz32 = exports.LfsType = void 0;
const tslib_1 = require("tslib");
const python_struct_1 = require("python-struct");
const node_assert_1 = tslib_1.__importDefault(require("node:assert"));
const node_util_1 = require("node:util");
const node_events_1 = require("node:events");
/* Simple Littlefs writer implementation. Flat structure (no directories),
* all metadata fits into one commit (2048 bytes on imp201).
*/
// type3 in LFS spec
var LfsType;
(function (LfsType) {
LfsType[LfsType["REG"] = 1] = "REG";
LfsType[LfsType["SUPERBLOCK"] = 255] = "SUPERBLOCK";
LfsType[LfsType["INLINESTRUCT"] = 513] = "INLINESTRUCT";
LfsType[LfsType["CTZSTRUCT"] = 514] = "CTZSTRUCT";
LfsType[LfsType["CRC"] = 1280] = "CRC";
})(LfsType = exports.LfsType || (exports.LfsType = {}));
// 04c11db7 polynomial
function crc32(data, initCrc = 0xffffffff) {
let res = initCrc;
const table = [
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac,
0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c,
0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c,
];
for (const d of data) {
res = (res >>> 4) ^ table[(res ^ d) & 0xf];
res = (res >>> 4) ^ table[(res ^ (d >>> 4)) & 0xf];
}
// Here and elsewhere '>>> 0' is a way to tell JS to interpret a number as unsigned
return res >>> 0;
}
function ctz32(n) {
let res = 0;
for (; res < 32; res++) {
if (n & 1) {
break;
}
n = n >>> 1;
}
return res;
}
exports.ctz32 = ctz32;
function padToLength(data, requiredLength) {
if (requiredLength <= data.length) {
return data;
}
let res = new Uint8Array(requiredLength);
res.set(data);
res.fill(0xff, data.length);
return res;
}
class LittleFsParams {
constructor() {
this.version = 0x00020000;
this.blockSize = 4096;
this.progSize = 2048;
this.blockCount = 0;
this.nameMax = 255;
this.fileMax = 0x7fffffff;
this.attrMax = 0x3fe;
}
encode() {
return (0, python_struct_1.pack)('<LLLLLL', this.version, this.blockSize, this.blockCount, this.nameMax, this.fileMax, this.attrMax);
}
}
class LittleFsEncoder {
constructor(emitter, maxBlocks, params, files) {
this.errorEmitter = emitter;
this.params = params;
this.files = files;
this.currentTag = 0xffffffff;
this.superblock = new Uint8Array([0x01, 0x00, 0x00, 0x00]); // Revision
let blocksRequired = 8; // reserve 8 blocks for metadata
for (let [name, content] of this.files) {
// reserve an extra block per file to account for pointers
blocksRequired += this.ctzBlocksForSize(content.length);
if (blocksRequired > maxBlocks) {
this.errorEmitter.emit('error', new Error(`File "${name}" does not fit into the file system`));
this.blocks = new Array(16); // Make TS happy, we won't really need it
return;
}
}
this.params.blockCount = blocksRequired;
this.blocks = new Array(this.params.blockCount);
}
addMetadata(type3, body, id = 0, lengthField = undefined) {
if (typeof lengthField === 'undefined') {
lengthField = body.length;
}
if (lengthField > 0b1111111111) {
return false;
}
let tag = ((type3 << 20) | (id << 10) | lengthField) >>> 0;
this.superblock = new Uint8Array([...this.superblock, ...(0, python_struct_1.pack)('>L', (tag ^ this.currentTag) >>> 0), ...body]);
this.currentTag = tag;
return true;
}
findFreeBlock() {
// skip first 8 blocks to give space for metadata growth
for (let i = 8; i < this.blocks.length; i++) {
if (typeof this.blocks[i] === 'undefined') {
return i;
}
}
return undefined;
}
// In a skip-list of CTZ blocks each block I will have N pointers back, where N
// is the largest power of 2 in factorisation of I (calculated using CTZ instruction
// on the device or with its imitation here).
// This method calculated the amount of free space (total space minus the space needeed
// for pointers) in Ith block of the chain.
dataSpaceInCtzBlock(index) {
const numPointers = (index === 0) ? 0 : (ctz32(index) + 1);
return this.params.blockSize - numPointers * 4;
}
pointerIndicesForCtzBlock(index) {
const numPointers = (index === 0) ? 0 : (ctz32(index) + 1);
let res = new Array(numPointers);
for (let i = 0; i < numPointers; i++) {
res[i] = index - (1 << i);
}
return res;
}
encodePointers(blockIndex, blockPointers) {
// From littlefs SPEC:
// "For every nth block where n is divisible by 2^x, that block
// contains a pointer to block n-2^x. These pointers are stored in
// increasing order of x in each block of the file before the actual data"
//
// Note that every number except 0 is divisible by 2^0
let indices = this.pointerIndicesForCtzBlock(blockIndex);
let res = new Uint8Array(indices.length * 4);
for (let i = 0; i < indices.length; i++) {
res.set((0, python_struct_1.pack)('<L', blockPointers[indices[i]]), i * 4);
}
return res;
}
ctzBlocksForSize(size) {
let blocks = 0;
while (size > 0) {
size -= this.dataSpaceInCtzBlock(blocks);
blocks += 1;
}
return Math.max(1, blocks);
}
// returns pointer to the last block as required by CTZ metadata structure
writeContentToCtzBlocks(content) {
const numBlocksRequired = this.ctzBlocksForSize(content.length);
let blockPointers = new Array(numBlocksRequired);
for (let i = 0; i < numBlocksRequired; i++) {
const freeBlock = this.findFreeBlock();
if (typeof (freeBlock) === 'undefined') {
return undefined;
}
blockPointers[i] = freeBlock;
let chunkSize = Math.min(this.dataSpaceInCtzBlock(i), content.length);
let pointerBlock = this.encodePointers(i, blockPointers);
let blockData = new Uint8Array(this.params.blockSize);
blockData.set(pointerBlock);
blockData.set(content.slice(0, chunkSize), pointerBlock.length);
blockData.fill(0xff, pointerBlock.length + chunkSize);
this.blocks[blockPointers[i]] = blockData;
content = content.slice(chunkSize);
}
(0, node_assert_1.default)(content.length === 0);
return blockPointers.at(-1);
}
allocSuperblock() {
let encoder = new node_util_1.TextEncoder();
(0, node_assert_1.default)(this.addMetadata(LfsType.SUPERBLOCK, encoder.encode("littlefs")));
(0, node_assert_1.default)(this.addMetadata(LfsType.INLINESTRUCT, this.params.encode()));
}
encodeFile(name, content, index) {
let encoder = new node_util_1.TextEncoder();
if (!this.addMetadata(LfsType.REG, encoder.encode(name), index)) {
this.errorEmitter.emit('error', new Error(`File name "${name}" is too long`));
return;
}
const offset = this.writeContentToCtzBlocks(content);
(0, node_assert_1.default)(typeof offset !== 'undefined');
const structData = (0, python_struct_1.pack)('<LL', offset, content.length);
(0, node_assert_1.default)(this.addMetadata(LfsType.CTZSTRUCT, structData, index));
}
commit() {
// this is not documented (couldn't find anything) but it's how littlefs
// implementation behaves: multiple CRC tags are created for a commit,
// because padding in a single one is limited to 1023 bytes (maximum length
// 10-bit field can hold).
if (this.superblock.length > this.params.progSize - 8) {
this.errorEmitter.emit('error', new Error(`Metadata of size ${this.superblock.length} does not fit program block of size ${this.params.progSize}`));
}
let crc = crc32(this.superblock);
while (this.superblock.length != this.params.progSize) {
let numPaddingBytes = Math.min(this.params.progSize - this.superblock.length - 4, 0x3fe);
const tail = this.params.progSize - this.superblock.length - numPaddingBytes - 4;
if (tail != 0 && tail < 8) {
// No space for next CRC tag, reduce padding
numPaddingBytes -= 8;
}
// cannot use addMetadata here, because the tag should be CRC'd too
const tag = ((LfsType.CRC << 20) | (0x3ff << 10) | numPaddingBytes) >>> 0;
const tagBytes = (0, python_struct_1.pack)('>L', (tag ^ this.currentTag) >>> 0);
crc = crc32(tagBytes, crc);
let crcBody = new Uint8Array(numPaddingBytes);
crcBody.set((0, python_struct_1.pack)('<L', crc));
if (numPaddingBytes > 4) {
crcBody.fill(0xff, 4);
}
this.superblock = new Uint8Array([...this.superblock, ...tagBytes, ...crcBody]);
this.currentTag = tag;
crc = 0xffffffff;
}
// now fill the rest of the block with zeros
if (this.params.blockSize != this.params.progSize) {
let padding = new Uint8Array(this.params.blockSize - this.params.progSize);
padding.fill(0xff);
this.superblock = new Uint8Array([...this.superblock, ...padding]);
}
}
finaliseBlocks() {
this.blocks[0] = this.superblock;
this.blocks[1] = this.superblock;
for (let i = 0; i < this.blocks.length; i++) {
if (typeof this.blocks[i] === 'undefined') {
this.blocks[i] = new Uint8Array(this.params.blockSize);
this.blocks[i].fill(0xff);
}
}
}
blocksMerged() {
const totalLen = this.blocks.length * this.params.blockSize;
let res = new Uint8Array(totalLen);
let offset = 0;
for (let i = 0; i < this.blocks.length; i++) {
(0, node_assert_1.default)(typeof this.blocks[i] !== 'undefined');
const block = this.blocks[i];
res.set(block, offset);
offset += block.length;
}
return res;
}
encode() {
this.allocSuperblock();
let index = 1;
for (let [name, content] of this.files) {
this.encodeFile(name, content, index);
index++;
}
this.commit();
this.finaliseBlocks();
return this.blocksMerged();
}
}
class LittleFsWriter extends node_events_1.EventEmitter {
constructor(maxSize) {
super();
this.maxSize = maxSize;
this.params = new LittleFsParams();
this.files = new Map();
}
addFile(name, contents) {
if (this.files.has(name)) {
this.emit('error', new Error(`File with name "${name}" already exists`));
return;
}
this.files.set(name, contents);
}
encode() {
let encoder = new LittleFsEncoder(this, Math.floor(this.maxSize / this.params.blockSize), this.params, this.files);
return encoder.encode();
}
}
exports.LittleFsWriter = LittleFsWriter;