@dstanesc/o-o-o-o-o-o-o
Version:
O-O-O-O-O-O-O is a collection of content addressed persistent data structures
953 lines (861 loc) • 26.6 kB
text/typescript
import { fastCloneVersionDetails } from './clone'
import {
Vertex,
Edge,
Prop,
Ref,
Type,
PropValue,
Link,
Block,
Index,
IndexedValue,
Status,
Part,
Version,
VersionDetails,
Tag,
Offset,
ValueRef,
PropRef,
PropRecord,
} from './types'
const REF_EXISTS = 0x1
const TYPE_EXISTS = 0x1
const LINK_EXISTS = 0x1
class BinaryEncoder {
buffer: Uint8Array
cursor: number
constructor(size: number) {
this.buffer = new Uint8Array(size)
this.cursor = 0
}
content() {
return this.buffer.subarray(0, this.cursor) // identical w/ fixed size buffer
}
/*
* String, fixed size, first 4 bytes hold the actual size
*/
// writeStringFixedSize(value: string, size: number) { // size + 4
// const bytes = pack(value)
// if (bytes.length > size) throw new Error(`String too large - ${bytes.length}, max allowed ${size} bytes`)
// const length = bytes.length
// this.writeUInt(length)
// const fixedSizeBuffer = new Uint8Array(size)
// fixedSizeBuffer.set(bytes, 0)
// return this.writeBytes(fixedSizeBuffer)
// }
writeByte(byte: number) {
this.buffer[this.cursor] = byte
this.cursor += 1
}
writeBytes(bytes: Uint8Array) {
const start = this.cursor
this.buffer.set(bytes, start)
this.cursor += bytes.byteLength
return this.cursor
}
/*
* Unsigned 32-bit integer, little endian
*/
writeUInt(value: number) {
if (value < 0 || value > 0xffffffff)
throw new Error('Integer out of range')
const start = this.cursor
this.buffer[start] = value & 0xff
this.buffer[start + 1] = value >>> 8
this.buffer[start + 2] = value >>> 16
this.buffer[start + 3] = value >>> 24
this.cursor += 4
return this.cursor
}
/*
* 32-bit integer, little endian
*/
writeInt(value: number) {
if (value < -0x80000000 || value > 0x7fffffff)
throw new Error('Integer out of range')
const start = this.cursor
this.buffer[start] = value & 0xff
this.buffer[start + 1] = value >>> 8
this.buffer[start + 2] = value >>> 16
this.buffer[start + 3] = value >>> 24
this.cursor += 4
return this.cursor
}
writeReserved(flags: number) {
return this.writeByte(flags)
}
writeRefExists() {
let flags = 0
flags |= REF_EXISTS
return this.writeByte(flags)
}
writeOffset(offset: number) {
//if (this.cursor !== offset) throw new Error(`Writing invalid offset ${offset}, current offset is ${this.cursor}`)
return this.writeUInt(offset)
}
writeRef(ref: Ref) {
// 5
this.writeRefExists() // 1
return this.writeUInt(ref) // 4
}
writeTypeExists() {
let flags = 0
flags |= TYPE_EXISTS
return this.writeByte(flags)
}
writeStatus(flag: Status) {
let flags = 0
flags |= flag
return this.writeByte(flags)
}
writeType(type: Type) {
//5
this.writeTypeExists() // 1
return this.writeUInt(type) // 4
}
writeLinkExists() {
let flags = 0
flags |= LINK_EXISTS
return this.writeByte(flags)
}
writeLink(link: Link) {
this.writeBytes(link.bytes)
}
skipBytes(length: number) {
this.cursor += length
return this.cursor
}
skipReserved() {
return this.skipBytes(1)
}
skipUInt() {
// 4
return this.skipBytes(4)
}
skipRef() {
// 5
return this.skipBytes(5)
}
skipType() {
// 5
return this.skipBytes(5)
}
}
class BinaryDecoder {
buffer: Uint8Array
cursor: number
constructor(buffer: Uint8Array) {
this.buffer = buffer
this.cursor = 0
}
readByte() {
const start = this.cursor
this.cursor += 1
return this.buffer[start]
}
readBytes(length: number) {
const start = this.cursor
this.cursor += length
const bytes = this.buffer.subarray(start, this.cursor)
return bytes
}
// readStringFixedSize(bufferSize: number) {
// const stringSize = this.readUInt()
// const stringBytes = this.readBytes(stringSize)
// this.cursor += bufferSize - stringSize
// return unpack(stringBytes)
// }
/*
* Unsigned 32-bit integer, little endian
*/
readUInt() {
const start = this.cursor
const value =
(this.buffer[start] |
(this.buffer[start + 1] << 8) |
(this.buffer[start + 2] << 16)) +
this.buffer[start + 3] * 0x1000000
this.cursor += 4
return value
}
/*
* 32-bit integer, little endian
*/
readInt() {
const start = this.cursor
const value =
this.buffer[start] |
(this.buffer[start + 1] << 8) |
(this.buffer[start + 2] << 16) |
(this.buffer[start + 3] << 24)
this.cursor += 4
return value
}
readOffset() {
return this.readUInt()
}
readRefExists() {
// 1
return this.readByte()
}
readTypeExists() {
// 1
return this.readByte()
}
readLinkExists() {
// 1
return this.readByte()
}
readStatus(): Status {
// 1
const flags = this.readByte()
return flags as Status
}
readRef() {
// 5
const flags = this.readRefExists() // 1
if (flags & REF_EXISTS) {
return this.readOffset() // 4
} else {
this.skipUInt()
return undefined
}
}
readType() {
// 5
const flags = this.readTypeExists() // 1
if (flags & TYPE_EXISTS) {
return this.readUInt() // 4
} else {
this.skipUInt()
return undefined
}
}
readLink(decode: (bytes: Uint8Array) => Link) {
const bytes = this.readBytes(36)
return decode(bytes)
}
readOptionalLink(decode: (bytes: Uint8Array) => Link): Link | undefined {
const flags = this.readLinkExists() // 1
if (flags & LINK_EXISTS) {
return this.readLink(decode)
} else {
this.skipBytes(36)
return undefined
}
}
skipBytes(length: number) {
this.cursor += length
}
skipReserved() {
// 1
return this.skipBytes(1)
}
skipUInt() {
// 4
return this.skipBytes(4)
}
skipRef() {
// 5
return this.skipBytes(5)
}
skipType() {
// 5
return this.skipBytes(5)
}
}
const VERTEX_SIZE_BYTES = 25
class VertexEncoder extends BinaryEncoder {
vertices: Vertex[]
constructor(vertices: Vertex[]) {
super(vertices.length * VERTEX_SIZE_BYTES)
this.vertices = vertices
}
writeVertex(vertex: Vertex) {
// 20
this.writeOffset(vertex.offset) // 4
if (vertex.type !== undefined)
// 5
this.writeType(vertex.type)
else this.skipType()
if (vertex.nextEdge !== undefined)
// 5
this.writeRef(vertex.nextEdge)
else this.skipRef()
if (vertex.nextProp !== undefined)
// 5
this.writeRef(vertex.nextProp)
else this.skipRef()
if (vertex.nextIndex !== undefined)
// 5
this.writeRef(vertex.nextIndex)
else this.skipRef()
this.writeStatus(vertex.status) // 1
}
write() {
for (const vertex of this.vertices) this.writeVertex(vertex)
return this.content()
}
}
class VertexDecoder extends BinaryDecoder {
readVertex(): Vertex {
const offset = this.readOffset()
const type = this.readType()
const nextEdge = this.readRef()
const nextProp = this.readRef()
const nextIndex = this.readRef()
const status = this.readStatus()
const vertex: Vertex = { status, offset }
if (type !== undefined) vertex.type = type
if (nextEdge !== undefined) vertex.nextEdge = nextEdge
if (nextProp !== undefined) vertex.nextProp = nextProp
if (nextIndex !== undefined) vertex.nextIndex = nextIndex
return vertex
}
read(): Vertex[] {
if (this.buffer.byteLength % VERTEX_SIZE_BYTES !== 0)
throw new Error('Invalid vertex serialization')
const size = Math.trunc(this.buffer.byteLength / VERTEX_SIZE_BYTES)
const vertices = []
for (let i = 0; i < size; i++) {
vertices.push(this.readVertex())
}
return vertices
}
}
const EDGE_SIZE_BYTES = 45
class EdgeEncoder extends BinaryEncoder {
edges: Edge[]
constructor(edges: Edge[]) {
super(edges.length * EDGE_SIZE_BYTES)
this.edges = edges
}
writeEdge(edge: Edge) {
this.writeOffset(edge.offset) // 4
if (edge.type !== undefined)
// 5
this.writeType(edge.type)
else this.skipType()
this.writeRef(edge.source) // 5
this.writeRef(edge.target) // 5
if (edge.sourcePrev !== undefined)
// 5
this.writeRef(edge.sourcePrev)
else this.skipRef()
if (edge.sourceNext !== undefined)
// 5
this.writeRef(edge.sourceNext)
else this.skipRef()
if (edge.targetPrev !== undefined)
// 5
this.writeRef(edge.targetPrev)
else this.skipRef()
if (edge.targetNext !== undefined)
// 5
this.writeRef(edge.targetNext)
else this.skipRef()
if (edge.nextProp !== undefined)
// 5
this.writeRef(edge.nextProp)
else this.skipRef()
this.writeStatus(edge.status) // 1
}
write() {
for (const edge of this.edges) this.writeEdge(edge)
return this.content()
}
}
class EdgeDecoder extends BinaryDecoder {
readEdge(): Edge {
const offset = this.readOffset()
const type = this.readType()
const source = this.readRef()
const target = this.readRef()
const sourcePrev = this.readRef()
const sourceNext = this.readRef()
const targetPrev = this.readRef()
const targetNext = this.readRef()
const nextProp = this.readRef()
const status = this.readStatus()
const edge: Edge = { status, offset, source, target }
if (type !== undefined) edge.type = type
if (sourcePrev !== undefined) edge.sourcePrev = sourcePrev
if (sourceNext !== undefined) edge.sourceNext = sourceNext
if (targetPrev !== undefined) edge.targetPrev = targetPrev
if (targetNext !== undefined) edge.targetNext = targetNext
if (nextProp !== undefined) edge.nextProp = nextProp
return edge
}
read(): Edge[] {
if (this.buffer.byteLength % EDGE_SIZE_BYTES !== 0)
throw new Error('Invalid edge serialization')
const size = Math.trunc(this.buffer.byteLength / EDGE_SIZE_BYTES)
const edges = []
for (let i = 0; i < size; i++) {
edges.push(this.readEdge())
}
return edges
}
}
const PROP_SIZE_BYTES = 32
class PropEncoder extends BinaryEncoder {
props: Prop[]
refs: Map<PropRef, ValueRef>
constructor(props: Prop[], refs: Map<PropRef, ValueRef>) {
super(props.length * PROP_SIZE_BYTES)
this.props = props
this.refs = refs
}
async writeValue(propRef: PropRef, value: PropValue) {
// 13
const valueRef: ValueRef = this.refs.get(propRef)
this.writeOffset(propRef) // 4
if (valueRef !== undefined) {
this.writeRef(valueRef.ref) // 5
this.writeUInt(valueRef.length) // 4
} else {
this.skipRef()
this.skipUInt()
}
}
async writeProp(prop: Prop) {
// 32
this.writeOffset(prop.offset) // 4
if (prop.type !== undefined)
// 5
this.writeType(prop.type)
else this.skipType()
this.writeUInt(prop.key) // 4
await this.writeValue(prop.offset, prop.value) // 13
if (prop.nextProp !== undefined)
// 5
this.writeRef(prop.nextProp)
else this.skipRef()
this.writeStatus(prop.status) // 1
}
async write() {
for (const prop of this.props) await this.writeProp(prop)
return this.content()
}
}
class PropRecordDecoder extends BinaryDecoder {
constructor(buffer: Uint8Array) {
super(buffer)
}
async readPropValueRef(): Promise<ValueRef> {
const propRef = this.readOffset()
const ref = this.readRef()
if (ref !== undefined) {
const length = this.readUInt()
const valueRef: ValueRef = { propRef, ref, length }
return valueRef
} else {
this.skipUInt()
return undefined
}
}
async readProp(): Promise<PropRecord> {
const offset = this.readOffset() //4
const type = this.readType() //5
const key = this.readUInt() //4
const valueRef = await this.readPropValueRef() // 13
const nextProp = this.readRef() //5
const status = this.readStatus() //1
const propRecord: PropRecord = { status, offset, key, valueRef }
if (type !== undefined) propRecord.type = type
if (nextProp !== undefined) propRecord.nextProp = nextProp
return propRecord
}
async read(): Promise<PropRecord[]> {
if (this.buffer.byteLength % PROP_SIZE_BYTES !== 0)
throw new Error('Invalid prop serialization')
const size = Math.trunc(this.buffer.byteLength / PROP_SIZE_BYTES)
const props = []
for (let i = 0; i < size; i++) {
props.push(await this.readProp())
}
return props
}
}
class PropDecoder extends BinaryDecoder {
valueGet: (ref: ValueRef) => Promise<PropValue>
constructor(
buffer: Uint8Array,
valueGet: (ref: ValueRef) => Promise<PropValue>
) {
super(buffer)
this.valueGet = valueGet
}
async readPropValue(): Promise<PropValue> {
const propRef = this.readOffset()
const ref = this.readRef()
if (ref !== undefined) {
const length = this.readUInt()
const valueRef: ValueRef = { propRef, ref, length }
return await this.valueGet(valueRef)
} else {
this.skipUInt()
return undefined
}
}
async readProp(): Promise<Prop> {
const offset = this.readOffset() //4
const type = this.readType() //5
const key = this.readUInt() //4
const value = await this.readPropValue() // 13
const nextProp = this.readRef() //5
const status = this.readStatus() //1
const prop: Prop = { status, offset, key, value }
if (type !== undefined) prop.type = type
if (nextProp !== undefined) prop.nextProp = nextProp
return prop
}
async read(): Promise<Prop[]> {
if (this.buffer.byteLength % PROP_SIZE_BYTES !== 0)
throw new Error('Invalid prop serialization')
const size = Math.trunc(this.buffer.byteLength / PROP_SIZE_BYTES)
const props = []
for (let i = 0; i < size; i++) {
props.push(await this.readProp())
}
return props
}
}
class PropValueEncoder extends BinaryEncoder {
props: Prop[]
valueEncode: (json: any) => Uint8Array
refOffset: number
constructor(
refOffset: number,
props: Prop[],
valueEncode: (json: any) => Uint8Array
) {
super(
props
.map((prop) => valueEncode(prop.value).byteLength)
.reduce((a, b) => a + b, 0)
)
this.refOffset = refOffset
this.props = props
this.valueEncode = valueEncode
}
writeValue(propRef: Ref, value: PropValue): ValueRef {
const bytes: Uint8Array = this.valueEncode(value)
const ref = this.cursor + this.refOffset
const length = bytes.byteLength
this.writeBytes(bytes)
return { propRef, ref, length }
}
writeProp(prop: Prop): ValueRef {
const valueRef: ValueRef = this.writeValue(prop.offset, prop.value)
return valueRef
}
write(): { buf: Uint8Array; refs: Map<PropRef, ValueRef> } {
const refs: Map<PropRef, ValueRef> = new Map()
for (const prop of this.props) {
const valueRef: ValueRef = this.writeProp(prop)
refs.set(prop.offset, valueRef)
}
const buf = this.content()
return { buf, refs }
}
}
class PropValueDecoder extends BinaryDecoder {
valueDecode: (bytes: Uint8Array) => any
constructor(buffer: Uint8Array, valueDecode: (bytes: Uint8Array) => any) {
super(buffer)
this.valueDecode = valueDecode
}
readValue(valueRef: ValueRef): PropValue {
const bytes = this.readBytes(valueRef.length)
return this.valueDecode(bytes)
}
read(valueRefs: ValueRef[]): Map<PropRef, PropValue> {
const values: Map<PropRef, PropValue> = new Map()
for (const valueRef of valueRefs) {
const value = this.readValue(valueRef)
values.set(valueRef.propRef, value)
}
return values
}
}
const INDEX_SIZE_BYTES = 56
class IndexEncoder extends BinaryEncoder {
indices: Index[]
constructor(indices: Index[]) {
super(indices.length * INDEX_SIZE_BYTES)
this.indices = indices
}
async writeIndex(index: Index) {
// 56
this.writeOffset(index.offset) // 4
if (index.type !== undefined)
// 5
this.writeType(index.type)
else this.skipType()
this.writeType(index.key) // 5
this.writeLink(index.value) // 36
if (index.nextIndex !== undefined)
// 5
this.writeRef(index.nextIndex)
else this.skipRef()
this.writeStatus(index.status) // 1
}
async write() {
for (const index of this.indices) await this.writeIndex(index)
return this.content()
}
}
class IndexDecoder extends BinaryDecoder {
linkDecode: (linkBytes: Uint8Array) => Link
indexDecode: (
link: Link,
blockGet: (cid: any) => Promise<Uint8Array>,
value?: any
) => Promise<IndexedValue[]>
blockGet: (cid: any) => Promise<Uint8Array>
constructor(
buffer: Uint8Array,
linkDecode: (linkBytes: Uint8Array) => Link
) {
super(buffer)
this.linkDecode = linkDecode
}
async readIndex(): Promise<Index> {
const offset = this.readOffset()
const type = this.readType()
const key = this.readType()
const value = this.readLink(this.linkDecode)
const nextIndex = this.readRef()
const status = this.readStatus()
const index: Index = { status, offset, key, value }
if (type !== undefined) index.type = type
if (nextIndex !== undefined) index.nextIndex = nextIndex
return index
}
async read(): Promise<Index[]> {
if (this.buffer.byteLength % INDEX_SIZE_BYTES !== 0)
throw new Error('Invalid index serialization')
const size = Math.trunc(this.buffer.byteLength / INDEX_SIZE_BYTES)
const indices = []
for (let i = 0; i < size; i++) {
indices.push(await this.readIndex())
}
return indices
}
}
class RootEncoder extends BinaryEncoder {
vertexRoot: Link // 36
vertexOffset: number // 4
edgeRoot: Link // 36
edgeOffset: number // 4
propRoot: Link // 36
propOffset: number // 4
valueRoot: Link // 36
valueOffset: number // 4
indexRoot: Link // 36
indexOffset: number // 4
constructor({
vertexRoot,
vertexOffset,
edgeRoot,
edgeOffset,
propRoot,
propOffset,
valueRoot,
valueOffset,
indexRoot,
indexOffset,
}: {
vertexRoot: Link
vertexOffset: number
edgeRoot: Link
edgeOffset: number
propRoot: Link
propOffset: number
valueRoot: Link
valueOffset: number
indexRoot: Link
indexOffset: number
}) {
super(200)
this.vertexRoot = vertexRoot
this.vertexOffset = vertexOffset
this.edgeRoot = edgeRoot
this.edgeOffset = edgeOffset
this.propRoot = propRoot
this.propOffset = propOffset
this.valueRoot = valueRoot
this.valueOffset = valueOffset
this.indexRoot = indexRoot
this.indexOffset = indexOffset
}
write() {
this.writeUInt(this.vertexOffset) //4
this.writeUInt(this.edgeOffset) //4
this.writeUInt(this.propOffset) //4
this.writeUInt(this.valueOffset) //4
this.writeUInt(this.indexOffset) //4
this.writeLink(this.vertexRoot as Link) //36
this.writeLink(this.edgeRoot as Link) //36
this.writeLink(this.propRoot as Link) //36
this.writeLink(this.valueRoot as Link) //36
this.writeLink(this.indexRoot as Link) //36
return this
}
}
class RootDecoder extends BinaryDecoder {
linkDecode: (bytes: Uint8Array) => Link
constructor(buffer: Uint8Array, linkDecode: (bytes: Uint8Array) => Link) {
super(buffer)
this.linkDecode = linkDecode
}
read(): {
vertexRoot: Link
vertexOffset: number
edgeRoot: Link
edgeOffset: number
propRoot: Link
propOffset: number
valueRoot: Link
valueOffset: number
indexRoot: Link
indexOffset: number
} {
const vertexOffset = this.readOffset()
const edgeOffset = this.readOffset()
const propOffset = this.readOffset()
const valueOffset = this.readOffset()
const indexOffset = this.readOffset()
const vertexRoot = this.readLink(this.linkDecode)
const edgeRoot = this.readLink(this.linkDecode)
const propRoot = this.readLink(this.linkDecode)
const valueRoot = this.readLink(this.linkDecode)
const indexRoot = this.readLink(this.linkDecode)
return {
vertexRoot,
vertexOffset,
edgeRoot,
edgeOffset,
propRoot,
propOffset,
valueRoot,
valueOffset,
indexRoot,
indexOffset,
}
}
}
const VERSION_CONST_SIZE_BYTES = 114 // 77
const VERSION_ID_SIZE_BYTES = 36
class VersionEncoder extends BinaryEncoder {
id: Link
versions: Version[]
valueEncode: (json: any) => Uint8Array
constructor(
id: Link,
versions: Version[],
valueEncode: (json: any) => Uint8Array
) {
super(
VERSION_ID_SIZE_BYTES +
versions
.map(
(version) =>
valueEncode(version.details).byteLength +
VERSION_CONST_SIZE_BYTES
)
.reduce((a, b) => a + b, 0)
)
this.id = id
this.versions = versions
this.valueEncode = valueEncode
}
writeDetails(details: VersionDetails) {
const purgedDetails = fastCloneVersionDetails(details)
const bytes: Uint8Array = this.valueEncode(purgedDetails)
const length = bytes.byteLength
this.writeUInt(length) // 4
this.writeBytes(bytes) // n
}
async writeVersion(version: Version) {
this.writeLink(version.root) // 36
if (version.parent !== undefined) {
this.writeLinkExists() // 1
this.writeLink(version.parent) // 36
} else this.skipBytes(37)
if (version.mergeParent !== undefined) {
this.writeLinkExists() // 1
this.writeLink(version.mergeParent) // 36
} else this.skipBytes(37)
this.writeDetails(version.details) // 4 + n
}
write() {
this.writeLink(this.id) // 36
for (const version of this.versions) this.writeVersion(version)
return this.content()
}
}
class VersionDecoder extends BinaryDecoder {
linkDecode: (linkBytes: Uint8Array) => Link
valueDecode: (bytes: Uint8Array) => any
constructor(
buffer: Uint8Array,
linkDecode: (linkBytes: Uint8Array) => Link,
valueDecode: (bytes: Uint8Array) => any
) {
super(buffer)
this.linkDecode = linkDecode
this.valueDecode = valueDecode
}
readVersionDetails(): VersionDetails {
const length = this.readUInt()
const bytes = this.readBytes(length)
return fastCloneVersionDetails(this.valueDecode(bytes))
}
readVersion(): Version {
const root = this.readLink(this.linkDecode)
const parent = this.readOptionalLink(this.linkDecode)
const mergeParent = this.readOptionalLink(this.linkDecode)
const details = this.readVersionDetails()
const version: Version = { root, details }
if (parent !== undefined) version.parent = parent
if (mergeParent !== undefined) version.mergeParent = mergeParent
return version
}
read(): { id: Link; versions: Version[] } {
const id: Link = this.readLink(this.linkDecode)
const versions = []
while (this.cursor < this.buffer.byteLength)
versions.push(this.readVersion())
return { id, versions }
}
}
const OFFSET_INCREMENTS = {
VERTEX_INCREMENT: VERTEX_SIZE_BYTES,
EDGE_INCREMENT: EDGE_SIZE_BYTES,
PROP_INCREMENT: PROP_SIZE_BYTES,
INDEX_INCREMENT: INDEX_SIZE_BYTES,
}
export {
BinaryEncoder,
BinaryDecoder,
VertexEncoder,
VertexDecoder,
EdgeEncoder,
EdgeDecoder,
PropRecordDecoder,
PropEncoder,
PropDecoder,
PropValueEncoder,
PropValueDecoder,
IndexEncoder,
IndexDecoder,
RootEncoder,
RootDecoder,
VersionEncoder,
VersionDecoder,
OFFSET_INCREMENTS,
}