@dstanesc/o-o-o-o-o-o-o
Version:
O-O-O-O-O-O-O is a collection of content addressed persistent data structures
744 lines • 26.4 kB
JavaScript
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
import { protoGremlinFactory } from '../api/proto-gremlin';
import * as assert from 'assert';
import bent from 'bent';
import { compute_chunks } from '@dstanesc/wasm-chunking-fastcdc-node';
import { chunkerFactory } from '../chunking';
import { memoryBlockStoreFactory, } from '../block-store';
import { linkCodecFactory, valueCodecFactory, } from '../codecs';
import { indexStoreFactory } from '../index-store-factory';
import { eq } from '../ops';
import { versionStoreFactory } from '../version-store';
import { navigateVertices, PathElemType, RequestBuilder } from '../navigate';
const getStream = bent('https://raw.githubusercontent.com');
var ObjectTypes;
(function (ObjectTypes) {
ObjectTypes[ObjectTypes["ROOT"] = 1] = "ROOT";
ObjectTypes[ObjectTypes["BOOK"] = 2] = "BOOK";
ObjectTypes[ObjectTypes["CHAPTER"] = 3] = "CHAPTER";
ObjectTypes[ObjectTypes["VERSE"] = 4] = "VERSE";
})(ObjectTypes || (ObjectTypes = {}));
var RlshpTypes;
(function (RlshpTypes) {
RlshpTypes[RlshpTypes["book"] = 1] = "book";
RlshpTypes[RlshpTypes["chapter"] = 2] = "chapter";
RlshpTypes[RlshpTypes["verse"] = 3] = "verse";
})(RlshpTypes || (RlshpTypes = {}));
var PropTypes;
(function (PropTypes) {
PropTypes[PropTypes["ANY"] = 1] = "ANY";
})(PropTypes || (PropTypes = {}));
var KeyTypes;
(function (KeyTypes) {
KeyTypes[KeyTypes["ID"] = 1] = "ID";
KeyTypes[KeyTypes["NAME"] = 2] = "NAME";
KeyTypes[KeyTypes["TEXT"] = 3] = "TEXT";
})(KeyTypes || (KeyTypes = {}));
var IndexTypes;
(function (IndexTypes) {
IndexTypes[IndexTypes["BOOK_ID"] = 2] = "BOOK_ID";
IndexTypes[IndexTypes["CHAPTER_ID"] = 3] = "CHAPTER_ID";
IndexTypes[IndexTypes["VERSE_ID"] = 4] = "VERSE_ID";
})(IndexTypes || (IndexTypes = {}));
describe('e2e ', function () {
test('full bible, 7MB json, no index, load and navigate', async () => {
const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
const str = (await stream.text()).trim();
const lines = str.split(/\r?\n/g);
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
const blockStore = memoryBlockStoreFactory();
const versionStore = await versionStoreFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
});
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
const tx = await g.tx();
const bible = await tx.addV(ObjectTypes.ROOT).next();
let book, book_id;
let chapter, chapter_id;
let verse, verse_id;
for (const line of lines) {
const entry = JSON.parse(line);
if (book === undefined || book_id !== entry.book_id) {
book = await tx
.addV(ObjectTypes.BOOK)
.property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
.property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
.next();
book_id = entry.book_id;
await tx.addE(RlshpTypes.book).from(bible).to(book).next();
}
if (chapter === undefined || chapter_id !== entry.chapter) {
chapter = await tx
.addV(ObjectTypes.CHAPTER)
.property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
.next();
chapter_id = entry.chapter;
await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
}
if (verse === undefined || verse_id !== entry.verse) {
verse = await tx
.addV(ObjectTypes.VERSE)
.property(KeyTypes.ID, entry.verse, PropTypes.ANY)
.property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
.next();
verse_id = entry.verse;
await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
}
}
const { root, index, blocks } = await tx.commit({});
const g2 = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
blockStore.resetReads();
const first = await queryVerse(g2, bible.offset, 'Gen', 1, 1);
const readsFirst = blockStore.countReads();
console.log(`Quick scan reads ${readsFirst}`);
assert.equal(first, 'In the beginning God created the heaven and the earth.');
blockStore.resetReads();
const last = await queryVerse(g2, bible.offset, 'Rev', 22, 21);
const readsLast = blockStore.countReads();
console.log(`Full scan reads ${readsLast}`);
assert.equal(last, 'The grace of our Lord Jesus Christ be with you all. Amen.');
console.log(`BlockStore total size = ${blockStore.size()}`);
console.log(`Root ${root.toString()} storeRoot=${versionStore
.versionStoreRoot()
.toString()}`);
/**
* Validate push
*/
const blockStore2 = memoryBlockStoreFactory();
await blockStore.push(blockStore2);
const versionStore2 = await versionStoreFactory({
readOnly: true,
storeRoot: versionStore.versionStoreRoot(),
versionRoot: undefined,
chunk,
linkCodec,
valueCodec,
blockStore: blockStore2,
});
const g3 = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore: blockStore2,
versionStore: versionStore2,
}).g();
const first2 = await queryVerse(g3, bible.offset, 'Gen', 1, 1);
assert.equal(first2, 'In the beginning God created the heaven and the earth.');
const last2 = await queryVerse(g3, bible.offset, 'Rev', 22, 21);
assert.equal(last2, 'The grace of our Lord Jesus Christ be with you all. Amen.');
assert.strictEqual(blockStore.size(), blockStore2.size());
const books = await queryBooks(g3, bible.offset);
});
test('full bible, 7MB json, no index, query first level', async () => {
const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
const str = (await stream.text()).trim();
const lines = str.split(/\r?\n/g);
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
const blockStore = memoryBlockStoreFactory();
const versionStore = await versionStoreFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
});
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
const tx = await g.tx();
const bible = await tx.addV(ObjectTypes.ROOT).next();
let book, book_id;
let chapter, chapter_id;
let verse, verse_id;
for (const line of lines) {
const entry = JSON.parse(line);
if (book === undefined || book_id !== entry.book_id) {
book = await tx
.addV(ObjectTypes.BOOK)
.property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
.property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
.next();
book_id = entry.book_id;
await tx.addE(RlshpTypes.book).from(bible).to(book).next();
}
if (chapter === undefined || chapter_id !== entry.chapter) {
chapter = await tx
.addV(ObjectTypes.CHAPTER)
.property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
.next();
chapter_id = entry.chapter;
await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
}
if (verse === undefined || verse_id !== entry.verse) {
verse = await tx
.addV(ObjectTypes.VERSE)
.property(KeyTypes.ID, entry.verse, PropTypes.ANY)
.property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
.next();
verse_id = entry.verse;
await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
}
}
const { root, index, blocks } = await tx.commit({});
const books = await queryBooks(g, bible.offset);
const expected = [
'Gen',
'Exod',
'Lev',
'Num',
'Deut',
'Josh',
'Judg',
'Ruth',
'1Sam',
'2Sam',
'1Kgs',
'2Kgs',
'1Chr',
'2Chr',
'Ezra',
'Neh',
'Esth',
'Job',
'Ps',
'Prov',
'Eccl',
'Song',
'Isa',
'Jer',
'Lam',
'Ezek',
'Dan',
'Hos',
'Joel',
'Amos',
'Obad',
'Jona',
'Mic',
'Nah',
'Hab',
'Zeph',
'Hag',
'Zech',
'Mal',
'Matt',
'Mark',
'Luke',
'John',
'Acts',
'Rom',
'1Cor',
'2Cor',
'Gal',
'Eph',
'Phil',
'Col',
'1Thess',
'2Thess',
'1Tim',
'2Tim',
'Titus',
'Phlm',
'Heb',
'Jas',
'1Pet',
'2Pet',
'1John',
'2John',
'3John',
'Jude',
'Rev',
];
assert.deepStrictEqual(books, expected);
});
test('full bible, 7MB json, KeyTypes.ID indexed, load and navigate', async () => {
const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
const str = (await stream.text()).trim();
const lines = str.split(/\r?\n/g);
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
const blockStore = memoryBlockStoreFactory();
const versionStore = await versionStoreFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
});
const indexStore = indexStoreFactory(blockStore);
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
indexStore,
}).g();
const tx = await g.tx();
const bible = await tx.addV(ObjectTypes.ROOT).next();
let book, book_id;
let chapter, chapter_id;
let verse, verse_id;
const books = [];
const chapters = [];
const verses = [];
for (const line of lines) {
const entry = JSON.parse(line);
if (book === undefined || book_id !== entry.book_id) {
book = await tx
.addV(ObjectTypes.BOOK)
.property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
.property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
.next();
book_id = entry.book_id;
await tx.addE(RlshpTypes.book).from(bible).to(book).next();
books.push(book);
}
if (chapter === undefined || chapter_id !== entry.chapter) {
chapter = await tx
.addV(ObjectTypes.CHAPTER)
.property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
.next();
chapter_id = entry.chapter;
await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
chapters.push(chapter);
}
if (verse === undefined || verse_id !== entry.verse) {
verse = await tx
.addV(ObjectTypes.VERSE)
.property(KeyTypes.ID, entry.verse, PropTypes.ANY)
.property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
.next();
verse_id = entry.verse;
await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
verses.push(verse);
}
}
for (const book of books) {
await book.uniqueIndex(KeyTypes.ID);
}
for (const chapter of chapters) {
await chapter.uniqueIndex(KeyTypes.ID);
}
for (const verse of verses) {
await verse.uniqueIndex(KeyTypes.ID);
}
const { root, index, blocks } = await tx.commit({});
const g2 = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
indexStore,
}).g();
blockStore.resetReads();
const first = await queryVerse(g2, bible.offset, 'Gen', 1, 1);
const readsFirst = blockStore.countReads();
console.log(`Indexed reads first ${readsFirst}`);
assert.equal(first, 'In the beginning God created the heaven and the earth.');
blockStore.resetReads();
const last = await queryVerse(g2, bible.offset, 'Rev', 22, 21);
const readsLast = blockStore.countReads();
console.log(`Indexed reads last ${readsLast}`);
assert.equal(last, 'The grace of our Lord Jesus Christ be with you all. Amen.');
console.log(`BlockStore total size = ${blockStore.size()}`);
console.log(`Root ${root.toString()} storeRoot=${versionStore
.versionStoreRoot()
.toString()}`);
});
test('full bible, 7MB json, no index, template based retrieval', async () => {
const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
const str = (await stream.text()).trim();
const lines = str.split(/\r?\n/g);
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
const blockStore = memoryBlockStoreFactory();
const versionStore = await versionStoreFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
});
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
const tx = await g.tx();
const bible = await tx.addV(ObjectTypes.ROOT).next();
let book, book_id;
let chapter, chapter_id;
let verse, verse_id;
for (const line of lines) {
const entry = JSON.parse(line);
if (book === undefined || book_id !== entry.book_id) {
book = await tx
.addV(ObjectTypes.BOOK)
.property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
.property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
.next();
book_id = entry.book_id;
await tx.addE(RlshpTypes.book).from(bible).to(book).next();
}
if (chapter === undefined || chapter_id !== entry.chapter) {
chapter = await tx
.addV(ObjectTypes.CHAPTER)
.property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
.next();
chapter_id = entry.chapter;
await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
}
if (verse === undefined || verse_id !== entry.verse) {
verse = await tx
.addV(ObjectTypes.VERSE)
.property(KeyTypes.ID, entry.verse, PropTypes.ANY)
.property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
.next();
verse_id = entry.verse;
await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
}
}
const { root, index, blocks } = await tx.commit({});
const books = await queryBooksTemplateGremlin(g, bible.offset, BOOKS_TEMPLATE_1);
assert.strictEqual(books.length, 66);
assert.strictEqual(books[0].id, 'Gen');
assert.strictEqual(books[0].name, 'Genesis');
assert.deepStrictEqual(books[0].chapters, [
{ id: 1 },
{ id: 2 },
{ id: 3 },
{ id: 4 },
{ id: 5 },
{ id: 6 },
{ id: 7 },
{ id: 8 },
{ id: 9 },
{ id: 10 },
{ id: 11 },
{ id: 12 },
{ id: 13 },
{ id: 14 },
{ id: 15 },
{ id: 16 },
{ id: 17 },
{ id: 18 },
{ id: 19 },
{ id: 20 },
{ id: 21 },
{ id: 22 },
{ id: 23 },
{ id: 24 },
{ id: 25 },
{ id: 26 },
{ id: 27 },
{ id: 28 },
{ id: 29 },
{ id: 30 },
{ id: 31 },
{ id: 32 },
{ id: 33 },
{ id: 34 },
{ id: 35 },
{ id: 36 },
{ id: 37 },
{ id: 38 },
{ id: 39 },
{ id: 40 },
{ id: 41 },
{ id: 42 },
{ id: 43 },
{ id: 44 },
{ id: 45 },
{ id: 46 },
{ id: 47 },
{ id: 48 },
{ id: 49 },
{ id: 50 },
]);
assert.strictEqual(books[1].id, 'Exod');
assert.strictEqual(books[1].name, 'Exodus');
assert.deepStrictEqual(books[1].chapters, [
{ id: 1 },
{ id: 2 },
{ id: 3 },
{ id: 4 },
{ id: 5 },
{ id: 6 },
{ id: 7 },
{ id: 8 },
{ id: 9 },
{ id: 10 },
{ id: 11 },
{ id: 12 },
{ id: 13 },
{ id: 14 },
{ id: 15 },
{ id: 16 },
{ id: 17 },
{ id: 18 },
{ id: 19 },
{ id: 20 },
{ id: 21 },
{ id: 22 },
{ id: 23 },
{ id: 24 },
{ id: 25 },
{ id: 26 },
{ id: 27 },
{ id: 28 },
{ id: 29 },
{ id: 30 },
{ id: 31 },
{ id: 32 },
{ id: 33 },
{ id: 34 },
{ id: 35 },
{ id: 36 },
{ id: 37 },
{ id: 38 },
{ id: 39 },
{ id: 40 },
]);
const books2 = await queryBooksTemplateGremlin(g, bible.offset, BOOKS_TEMPLATE_2);
assert.strictEqual(books2.length, 66);
assert.strictEqual(books2[0].id, 'Gen');
assert.strictEqual(books2[0].name, 'Genesis');
assert.deepStrictEqual(books2[0].chapters, [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
]);
assert.deepStrictEqual(books2[65], {
id: 'Rev',
name: 'Revelation',
chapters: [
2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22,
],
});
const books3 = await queryBooksTemplateNative(g.graph, bible.offset, BOOKS_TEMPLATE_2);
assert.strictEqual(books3.length, 66);
assert.deepStrictEqual(books3[0], {
id: 'Gen',
name: 'Genesis',
chapters: [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
],
});
assert.deepStrictEqual(books3[65], {
id: 'Rev',
name: 'Revelation',
chapters: [
2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22,
],
});
});
});
async function queryVerse(g, rootOffset, book, chapter, verse) {
var _a, e_1, _b, _c;
const vr = [];
const startTime = new Date().getTime();
try {
for (var _d = true, _e = __asyncValues(g
.V([rootOffset])
.out(RlshpTypes.book)
.has(ObjectTypes.BOOK, { keyType: KeyTypes.ID, operation: eq(book) })
.out(RlshpTypes.chapter)
.has(ObjectTypes.CHAPTER, {
keyType: KeyTypes.ID,
operation: eq(chapter),
})
.out(RlshpTypes.verse)
.has(ObjectTypes.VERSE, { keyType: KeyTypes.ID, operation: eq(verse) })
.values(KeyTypes.TEXT)
.maxResults(1)
.exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const result = _c;
vr.push(result);
}
finally {
_d = true;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
}
finally { if (e_1) throw e_1.error; }
}
const endTime = new Date().getTime();
console.log(`Query Time ${endTime - startTime} ms`);
return vr[0].value;
}
async function queryBooks(g, rootOffset) {
var _a, e_2, _b, _c;
const books = [];
try {
for (var _d = true, _e = __asyncValues(g
.V([rootOffset])
.out(RlshpTypes.book)
.hasType(ObjectTypes.BOOK)
.values(KeyTypes.ID)
.exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const result = _c;
books.push(result.value);
}
finally {
_d = true;
}
}
}
catch (e_2_1) { e_2 = { error: e_2_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
}
finally { if (e_2) throw e_2.error; }
}
return books;
}
const BOOKS_TEMPLATE_1 = {
id: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.ID,
},
name: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.NAME,
},
chapters: {
$elemType: PathElemType.EDGE,
$type: RlshpTypes.chapter,
id: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.ID,
},
},
};
const BOOKS_TEMPLATE_2 = {
id: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.ID,
},
name: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.NAME,
},
chapters: {
$elemType: PathElemType.EDGE,
$type: RlshpTypes.chapter,
$value: {
$elemType: PathElemType.EXTRACT,
$type: KeyTypes.ID,
},
},
};
async function queryBooksTemplateGremlin(g, rootOffset, template) {
var _a, e_3, _b, _c;
const vr = [];
const startTime = new Date().getTime();
try {
for (var _d = true, _e = __asyncValues(g
.V([rootOffset])
.out(RlshpTypes.book)
.hasType(ObjectTypes.BOOK)
.template(template)
.maxResults(100)
.exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const result = _c;
vr.push(result);
}
finally {
_d = true;
}
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
}
finally { if (e_3) throw e_3.error; }
}
const endTime = new Date().getTime();
return vr;
}
const queryBooksTemplateNative = async (graph, rootOffset, template) => {
var _a, e_4, _b, _c;
const request = new RequestBuilder()
.add(PathElemType.VERTEX)
.add(PathElemType.EDGE)
.add(PathElemType.VERTEX)
.template(template)
.maxResults(100)
.get();
const vr = [];
try {
for (var _d = true, _e = __asyncValues(navigateVertices(graph, [rootOffset], request)), _f; _f = await _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const result = _c;
vr.push(result);
}
finally {
_d = true;
}
}
}
catch (e_4_1) { e_4 = { error: e_4_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
}
finally { if (e_4) throw e_4.error; }
}
return vr;
};
//# sourceMappingURL=e2e.test.js.map