@atproto/repo
Version:
atproto repo and MST implementation
227 lines • 7.76 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.readCarStream = exports.readCarWithRoot = exports.readCar = exports.blocksToCarStream = exports.blocksToCarFile = void 0;
exports.writeCarStream = writeCarStream;
exports.verifyIncomingCarBlocks = verifyIncomingCarBlocks;
const cbor = __importStar(require("@ipld/dag-cbor"));
const ui8 = __importStar(require("uint8arrays"));
const varint = __importStar(require("varint"));
const common_1 = require("@atproto/common");
const block_map_1 = require("./block-map");
async function* writeCarStream(root, blocks) {
const header = new Uint8Array(cbor.encode({
version: 1,
roots: root ? [root] : [],
}));
yield new Uint8Array(varint.encode(header.byteLength));
yield header;
for await (const block of blocks) {
yield new Uint8Array(varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength));
yield block.cid.bytes;
yield block.bytes;
}
}
const blocksToCarFile = (root, blocks) => {
const carStream = (0, exports.blocksToCarStream)(root, blocks);
return (0, common_1.streamToBuffer)(carStream);
};
exports.blocksToCarFile = blocksToCarFile;
const blocksToCarStream = (root, blocks) => {
return writeCarStream(root, iterateBlocks(blocks));
};
exports.blocksToCarStream = blocksToCarStream;
async function* iterateBlocks(blocks) {
for (const entry of blocks.entries()) {
yield { cid: entry.cid, bytes: entry.bytes };
}
}
const readCar = async (bytes) => {
const { roots, blocks } = await (0, exports.readCarStream)([bytes]);
const blockMap = new block_map_1.BlockMap();
for await (const block of blocks) {
blockMap.set(block.cid, block.bytes);
}
return { roots, blocks: blockMap };
};
exports.readCar = readCar;
const readCarWithRoot = async (bytes) => {
const { roots, blocks } = await (0, exports.readCar)(bytes);
if (roots.length !== 1) {
throw new Error(`Expected one root, got ${roots.length}`);
}
const root = roots[0];
return {
root,
blocks,
};
};
exports.readCarWithRoot = readCarWithRoot;
const readCarStream = async (car) => {
const reader = new BufferedReader(car);
try {
const headerSize = await reader.readVarint();
if (headerSize === null) {
throw new Error('Could not parse CAR header');
}
const headerBytes = await reader.read(headerSize);
const header = cbor.decode(headerBytes);
if (!common_1.check.is(header, common_1.schema.carHeader)) {
throw new Error('Could not parse CAR header');
}
return {
roots: header.roots,
blocks: readCarBlocksIter(reader),
};
}
catch (err) {
await reader.close();
throw err;
}
};
exports.readCarStream = readCarStream;
const readCarBlocksIter = (reader) => {
const iter = readCarBlocksIterGenerator(reader);
iter.dump = async () => {
// try/finally to ensure that reader.close is called even if blocks.return throws.
try {
// Prevent the iterator from being started after this method is called.
await iter.return();
}
finally {
// @NOTE the "finally" block of the async generator won't be called
// if the iteration was never started so we need to manually close here.
await reader.close();
}
};
return iter;
};
async function* readCarBlocksIterGenerator(reader) {
try {
while (!reader.isDone) {
const blockSize = await reader.readVarint();
if (blockSize === null) {
break;
}
const blockBytes = await reader.read(blockSize);
const cid = (0, common_1.parseCidFromBytes)(blockBytes.subarray(0, 36));
const bytes = blockBytes.subarray(36);
yield { cid, bytes };
}
}
finally {
await reader.close();
}
}
async function* verifyIncomingCarBlocks(car) {
for await (const block of car) {
await (0, common_1.verifyCidForBytes)(block.cid, block.bytes);
yield block;
}
}
class BufferedReader {
constructor(stream) {
Object.defineProperty(this, "buffer", {
enumerable: true,
configurable: true,
writable: true,
value: new Uint8Array()
});
Object.defineProperty(this, "iterator", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "isDone", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
this.iterator =
Symbol.asyncIterator in stream
? stream[Symbol.asyncIterator]()
: stream[Symbol.iterator]();
}
async read(bytesToRead) {
await this.readUntilBuffered(bytesToRead);
const value = this.buffer.subarray(0, bytesToRead);
this.buffer = this.buffer.subarray(bytesToRead);
return value;
}
async readVarint() {
let done = false;
const bytes = [];
while (!done) {
const byte = await this.read(1);
if (byte.byteLength === 0) {
if (bytes.length > 0) {
throw new Error('could not parse varint');
}
else {
return null;
}
}
bytes.push(byte);
if (byte[0] < 128) {
done = true;
}
}
const concatted = ui8.concat(bytes);
return varint.decode(concatted);
}
async readUntilBuffered(bytesToRead) {
if (this.isDone) {
return;
}
while (this.buffer.length < bytesToRead) {
const next = await this.iterator.next();
if (next.done) {
this.isDone = true;
return;
}
this.buffer = ui8.concat([this.buffer, next.value]);
}
}
async close() {
if (!this.isDone && this.iterator.return) {
await this.iterator.return();
}
this.isDone = true;
this.buffer = new Uint8Array();
}
}
//# sourceMappingURL=car.js.map