@dstanesc/o-o-o-o-o-o-o/dist/__tests__/packer.test.js

O-O-O-O-O-O-O is a collection of content addressed persistent data structures

var __asyncValues = (this && this.__asyncValues) || function (o) {
    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
    var m = o[Symbol.asyncIterator], i;
    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
import { protoGremlinFactory } from '../api/proto-gremlin';
import { compute_chunks } from '@dstanesc/wasm-chunking-fastcdc-node';
import { chunkerFactory } from '../chunking';
import { memoryBlockStoreFactory, } from '../block-store';
import { create as ipfsApi } from 'ipfs-http-client';
import { blockStore as ipfsBlockStore } from '@dstanesc/ipfs-block-store';
import { linkCodecFactory, valueCodecFactory, } from '../codecs';
import { eq } from '../ops';
import * as assert from 'assert';
import bent from 'bent';
import { versionStoreFactory } from '../version-store';
import { graphPackerFactory } from '../graph-packer';
const getStream = bent('https://raw.githubusercontent.com');
var ObjectTypes;
(function (ObjectTypes) {
    ObjectTypes[ObjectTypes["ROOT"] = 1] = "ROOT";
    ObjectTypes[ObjectTypes["BOOK"] = 2] = "BOOK";
    ObjectTypes[ObjectTypes["CHAPTER"] = 3] = "CHAPTER";
    ObjectTypes[ObjectTypes["VERSE"] = 4] = "VERSE";
})(ObjectTypes || (ObjectTypes = {}));
var RlshpTypes;
(function (RlshpTypes) {
    RlshpTypes[RlshpTypes["book"] = 1] = "book";
    RlshpTypes[RlshpTypes["chapter"] = 2] = "chapter";
    RlshpTypes[RlshpTypes["verse"] = 3] = "verse";
})(RlshpTypes || (RlshpTypes = {}));
var KeyTypes;
(function (KeyTypes) {
    KeyTypes[KeyTypes["ID"] = 1] = "ID";
    KeyTypes[KeyTypes["NAME"] = 2] = "NAME";
    KeyTypes[KeyTypes["TEXT"] = 3] = "TEXT";
})(KeyTypes || (KeyTypes = {}));
var PropTypes;
(function (PropTypes) {
    PropTypes[PropTypes["ANY"] = 1] = "ANY";
})(PropTypes || (PropTypes = {}));
async function queryVerse(g, rootOffset, book, chapter, verse) {
    var _a, e_1, _b, _c;
    const vr = [];
    const startTime = new Date().getTime();
    try {
        for (var _d = true, _e = __asyncValues(g
            .V([rootOffset])
            .out(RlshpTypes.book)
            .has(ObjectTypes.BOOK, { keyType: KeyTypes.ID, operation: eq(book) })
            .out(RlshpTypes.chapter)
            .has(ObjectTypes.CHAPTER, {
            keyType: KeyTypes.ID,
            operation: eq(chapter),
        })
            .out(RlshpTypes.verse)
            .has(ObjectTypes.VERSE, { keyType: KeyTypes.ID, operation: eq(verse) })
            .values(KeyTypes.TEXT)
            .maxResults(1)
            .exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
            _c = _f.value;
            _d = false;
            try {
                const result = _c;
                vr.push(result);
            }
            finally {
                _d = true;
            }
        }
    }
    catch (e_1_1) { e_1 = { error: e_1_1 }; }
    finally {
        try {
            if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
        }
        finally { if (e_1) throw e_1.error; }
    }
    const endTime = new Date().getTime();
    const time = endTime - startTime;
    return { result: vr[0].value, time };
}
async function queryVerseFromBook(g, bookOffset, chapter, verse) {
    var _a, e_2, _b, _c;
    const vr = [];
    const startTime = new Date().getTime();
    try {
        for (var _d = true, _e = __asyncValues(g
            .V([bookOffset])
            .out(RlshpTypes.chapter)
            .has(ObjectTypes.CHAPTER, {
            keyType: KeyTypes.ID,
            operation: eq(chapter),
        })
            .out(RlshpTypes.verse)
            .has(ObjectTypes.VERSE, { keyType: KeyTypes.ID, operation: eq(verse) })
            .values(KeyTypes.TEXT)
            .maxResults(1)
            .exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
            _c = _f.value;
            _d = false;
            try {
                const result = _c;
                vr.push(result);
            }
            finally {
                _d = true;
            }
        }
    }
    catch (e_2_1) { e_2 = { error: e_2_1 }; }
    finally {
        try {
            if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
        }
        finally { if (e_2) throw e_2.error; }
    }
    const endTime = new Date().getTime();
    const time = endTime - startTime;
    return { result: vr[0].value, time };
}
async function quickVerse(g, verseOffset) {
    var _a, e_3, _b, _c;
    const vr = [];
    const startTime = new Date().getTime();
    try {
        for (var _d = true, _e = __asyncValues(g
            .V([verseOffset])
            .values(KeyTypes.TEXT)
            .maxResults(1)
            .exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
            _c = _f.value;
            _d = false;
            try {
                const result = _c;
                vr.push(result);
            }
            finally {
                _d = true;
            }
        }
    }
    catch (e_3_1) { e_3 = { error: e_3_1 }; }
    finally {
        try {
            if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
        }
        finally { if (e_3) throw e_3.error; }
    }
    const endTime = new Date().getTime();
    const time = endTime - startTime;
    return { result: vr[0].value, time };
}
describe('Graph packer', function () {
    test('pack and restore random blocks', async () => {
        const cache = {};
        const ipfs = ipfsApi({ url: process.env.IPFS_API });
        const linkCodec = linkCodecFactory();
        const blockStore = ipfsBlockStore({ cache, ipfs });
        const bytes1 = await blockStore.get('bafkreigu3fodriojosw3irmjz6v2znxli243n3xstcg575taxxjmctwu64');
        const bytes2 = await blockStore.get('bafkreihxekffa3coj4gpdmx2x5vteix5t5exrxfd7t4erqibj3mvw4rzm4');
        const bytes3 = await blockStore.get('bafkreib55oovsq4oxgiszoi43bsavb4raonyydlcyk6wx75hmb2ehkkikm');
        const block1 = {
            bytes: bytes1,
            cid: linkCodec.parseString('bafkreigu3fodriojosw3irmjz6v2znxli243n3xstcg575taxxjmctwu64'),
        };
        const block2 = {
            bytes: bytes2,
            cid: linkCodec.parseString('bafkreihxekffa3coj4gpdmx2x5vteix5t5exrxfd7t4erqibj3mvw4rzm4'),
        };
        const block3 = {
            bytes: bytes3,
            cid: linkCodec.parseString('bafkreib55oovsq4oxgiszoi43bsavb4raonyydlcyk6wx75hmb2ehkkikm'),
        };
        const { packRandomBlocks, restoreRandomBlocks } = graphPackerFactory(linkCodec);
        const bundle = await packRandomBlocks([block1, block2, block3]);
        const memStore = memoryBlockStoreFactory();
        const restored = await restoreRandomBlocks(bundle.bytes, memStore);
        expect(restored.length).toBe(3);
        expect(restored[0].cid.toString()).toBe('bafkreigu3fodriojosw3irmjz6v2znxli243n3xstcg575taxxjmctwu64');
        expect(restored[1].cid.toString()).toBe('bafkreihxekffa3coj4gpdmx2x5vteix5t5exrxfd7t4erqibj3mvw4rzm4');
        expect(restored[2].cid.toString()).toBe('bafkreib55oovsq4oxgiszoi43bsavb4raonyydlcyk6wx75hmb2ehkkikm');
    });
    test('pack and restore version store', async () => {
        const cache = {};
        const ipfs = ipfsApi({ url: process.env.IPFS_API });
        const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
        const linkCodec = linkCodecFactory();
        const valueCodec = valueCodecFactory();
        const blockStore = ipfsBlockStore({ cache, ipfs });
        const versionStoreRoot = linkCodec.parseString('bafkreiarwxmqgb7aroqi4wocldxcub25yclf6iz3ey4gmy4fqlnmkqvq3u');
        /*
         * Packing and restoring
         */
        const { packVersionStore, restoreSingleIndex: restoreVersionStore } = graphPackerFactory(linkCodec);
        const bundle = await packVersionStore(versionStoreRoot, blockStore, chunk, valueCodec);
        const memStore = memoryBlockStoreFactory();
        const { root: storeRoot } = await restoreVersionStore(bundle.bytes, memStore);
        const versionStore = await versionStoreFactory({
            storeRoot: storeRoot,
            chunk,
            linkCodec,
            valueCodec,
            blockStore: memStore,
        });
        assert.equal('bafkreielwa556n5ax2rngrplokqypcds2ypxnq5niufa4ls4l73flwt73i', versionStore.id());
        assert.equal('bafkreiarwxmqgb7aroqi4wocldxcub25yclf6iz3ey4gmy4fqlnmkqvq3u', versionStore.versionStoreRoot());
        assert.equal(1, versionStore.log().length);
        assert.equal('bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi', versionStore.currentRoot().toString());
    });
    test('pack and restore full graph', async () => {
        const cache = {};
        const ipfs = ipfsApi({ url: process.env.IPFS_API });
        const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
        const linkCodec = linkCodecFactory();
        const valueCodec = valueCodecFactory();
        const blockStore = ipfsBlockStore({ cache, ipfs });
        const versionRoot = linkCodec.parseString('bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
        const versionStore = await versionStoreFactory({
            versionRoot,
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
        });
        /*
         * Packing and restoring
         */
        const { packGraph, restore } = graphPackerFactory(linkCodec);
        const bundle = await packGraph(versionRoot, blockStore);
        const memStore = memoryBlockStoreFactory();
        const { root, index, blocks } = await restore(bundle.bytes, memStore);
        /**
         * Verifying
         */
        const g2 = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore: memStore,
            versionStore,
        }).g();
        const { result, time } = await queryVerse(g2, 0, 'Rev', 22, 21);
        assert.strictEqual(result, 'The grace of our Lord Jesus Christ be with you all. Amen.');
        assert.ok(time < 3000); // 3sec
        /**
         * More concise packing
         */
        const g = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
            versionStore,
        }).g();
        const bundle2 = await g.pack(versionRoot);
        assert.deepStrictEqual(bundle, bundle2);
    });
    test('pack and restore single commit', async () => {
        const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
        const str = (await stream.text()).trim();
        const lines = str.split(/\r?\n/g);
        const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
        const linkCodec = linkCodecFactory();
        const valueCodec = valueCodecFactory();
        const blockStore = memoryBlockStoreFactory();
        const versionStore = await versionStoreFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
        });
        const g = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
            versionStore,
        }).g();
        const tx = await g.tx();
        const bible = await tx.addV(ObjectTypes.ROOT).next();
        let book, book_id;
        let chapter, chapter_id;
        let verse, verse_id;
        for (const line of lines) {
            const entry = JSON.parse(line);
            if (book === undefined || book_id !== entry.book_id) {
                book = await tx
                    .addV(ObjectTypes.BOOK)
                    .property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
                    .property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
                    .next();
                book_id = entry.book_id;
                await tx.addE(RlshpTypes.book).from(bible).to(book).next();
            }
            if (chapter === undefined || chapter_id !== entry.chapter) {
                chapter = await tx
                    .addV(ObjectTypes.CHAPTER)
                    .property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
                    .next();
                chapter_id = entry.chapter;
                await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
            }
            if (verse === undefined || verse_id !== entry.verse) {
                verse = await tx
                    .addV(ObjectTypes.VERSE)
                    .property(KeyTypes.ID, entry.verse, PropTypes.ANY)
                    .property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
                    .next();
                verse_id = entry.verse;
                await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
            }
            // console.log(
            //     `Book:${entry.book_name}: ${book.offset} Chapter: ${entry.chapter}: ${chapter.offset} Verse: ${entry.verse}: ${verse.offset}`
            // )
        }
        const commit = await tx.commit({});
        const { packCommit, restore } = graphPackerFactory(linkCodec);
        const bundle = await packCommit(commit);
        const emptyStore = memoryBlockStoreFactory();
        const { root, index, blocks } = await restore(bundle.bytes, emptyStore);
        const g2 = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore: emptyStore,
            versionStore,
        }).g();
        {
            const { result, time } = await queryVerse(g2, 0, 'Rev', 22, 21);
            assert.strictEqual(result, 'The grace of our Lord Jesus Christ be with you all. Amen.');
            assert.ok(time < 3000); // 3sec
        }
    });
    test('pack and restore computed range, graph depth 1', async () => {
        const cache = {};
        const ipfs = ipfsApi({ url: process.env.IPFS_API });
        const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
        const linkCodec = linkCodecFactory();
        const valueCodec = valueCodecFactory();
        const blockStore = ipfsBlockStore({ cache, ipfs });
        const versionRoot = linkCodec.parseString('bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
        const versionStore = await versionStoreFactory({
            versionRoot,
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
        });
        const { packComputed, unpack, restore } = graphPackerFactory(linkCodec);
        const bundle = await packComputed(versionRoot, 808775, // first vertex ref
        1, // vertex count
        1, // graph depth
        blockStore, chunk, valueCodec);
        const emptyStore = memoryBlockStoreFactory();
        {
            const { root, index, blocks } = await restore(bundle.bytes, emptyStore);
            assert.strictEqual(root.toString(), 'bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
            assert.strictEqual(blocks.length, 9);
            const g2 = protoGremlinFactory({
                chunk,
                linkCodec,
                valueCodec,
                blockStore: emptyStore,
                versionStore,
            }).g();
            const { result, time } = await quickVerse(g2, 808800);
            assert.strictEqual(result, 'The grace of our Lord Jesus Christ be with you all. Amen.');
            assert.ok(time < 10); // 10ms
        }
        {
            const { root, index, blocks } = await unpack(bundle.bytes);
            assert.strictEqual(root.toString(), 'bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
            assert.strictEqual(blocks.length, 9);
            const otherStore = memoryBlockStoreFactory();
            for (const block of blocks) {
                await otherStore.put(block);
            }
            assert.deepStrictEqual(emptyStore.size(), otherStore.size());
        }
        {
            /**
             * More concise fragment packing
             */
            const g = protoGremlinFactory({
                chunk,
                linkCodec,
                valueCodec,
                blockStore,
                versionStore,
            }).g();
            const bundle2 = await g.packFragment(808775, 1, 1, versionRoot);
            assert.deepStrictEqual(bundle, bundle2);
        }
    });
    test('pack and restore computed range, graph depth 3', async () => {
        const cache = {};
        const ipfs = ipfsApi({ url: process.env.IPFS_API });
        const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
        const linkCodec = linkCodecFactory();
        const valueCodec = valueCodecFactory();
        const blockStore = ipfsBlockStore({ cache, ipfs });
        const versionRoot = linkCodec.parseString('bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
        const versionStore = await versionStoreFactory({
            versionRoot,
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
        });
        const g = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore,
            versionStore,
        }).g();
        // Packing single (Revelation) book, w/ depth 3 (book, chapters, verses) - vertex offset is 798175
        const bundle = await g.packFragment(798175, 1, 3, versionRoot);
        const emptyStore = memoryBlockStoreFactory();
        const { restore } = graphPackerFactory(linkCodec);
        await restore(bundle.bytes, emptyStore);
        const g2 = protoGremlinFactory({
            chunk,
            linkCodec,
            valueCodec,
            blockStore: emptyStore,
            versionStore,
        }).g();
        const { result: r1, time } = await queryVerseFromBook(g2, 798175, 22, 21);
        assert.strictEqual(r1, 'The grace of our Lord Jesus Christ be with you all. Amen.');
        const { result: r2 } = await queryVerseFromBook(g2, 798175, 22, 11);
        assert.strictEqual(r2, 'He that is unjust, let him be unjust still: and he which is filthy, let him be filthy still: and he that is righteous, let him be righteous still: and he that is holy, let him be holy still.');
    });
});
//# sourceMappingURL=packer.test.js.map