@dstanesc/o-o-o-o-o-o-o
Version:
O-O-O-O-O-O-O is a collection of content addressed persistent data structures
197 lines • 8.07 kB
JavaScript
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
import { protoGremlinFactory } from './api/proto-gremlin';
import bent from 'bent';
import { compute_chunks } from '@dstanesc/wasm-chunking-fastcdc-node';
import { chunkerFactory } from './chunking';
import { linkCodecFactory, valueCodecFactory, } from './codecs';
import { create as ipfsApi } from 'ipfs-http-client';
import { blockStore as ipfsBlockStore } from '@dstanesc/ipfs-block-store';
import { eq } from './ops';
import { versionStoreFactory } from './version-store';
const getStream = bent('https://raw.githubusercontent.com');
var ObjectTypes;
(function (ObjectTypes) {
ObjectTypes[ObjectTypes["ROOT"] = 1] = "ROOT";
ObjectTypes[ObjectTypes["BOOK"] = 2] = "BOOK";
ObjectTypes[ObjectTypes["CHAPTER"] = 3] = "CHAPTER";
ObjectTypes[ObjectTypes["VERSE"] = 4] = "VERSE";
})(ObjectTypes || (ObjectTypes = {}));
var RlshpTypes;
(function (RlshpTypes) {
RlshpTypes[RlshpTypes["book"] = 1] = "book";
RlshpTypes[RlshpTypes["chapter"] = 2] = "chapter";
RlshpTypes[RlshpTypes["verse"] = 3] = "verse";
})(RlshpTypes || (RlshpTypes = {}));
var PropTypes;
(function (PropTypes) {
PropTypes[PropTypes["ANY"] = 1] = "ANY";
})(PropTypes || (PropTypes = {}));
var KeyTypes;
(function (KeyTypes) {
KeyTypes[KeyTypes["ID"] = 1] = "ID";
KeyTypes[KeyTypes["NAME"] = 2] = "NAME";
KeyTypes[KeyTypes["TEXT"] = 3] = "TEXT";
})(KeyTypes || (KeyTypes = {}));
var IndexTypes;
(function (IndexTypes) {
IndexTypes[IndexTypes["BOOK_ID"] = 2] = "BOOK_ID";
IndexTypes[IndexTypes["CHAPTER_ID"] = 3] = "CHAPTER_ID";
IndexTypes[IndexTypes["VERSE_ID"] = 4] = "VERSE_ID";
})(IndexTypes || (IndexTypes = {}));
const stream = await getStream('/dstanesc/bibleapi-bibles-json/legacy/kjv.json');
const str = (await stream.text()).trim();
const lines = str.split(/\r?\n/g);
async function publish() {
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
const cache = {};
const ipfs = ipfsApi({ url: process.env.IPFS_API }); // eg. /ip4/192.168.1.231/tcp/5001
const blockStore = ipfsBlockStore({ cache, ipfs });
// const awsRegion = process.env.AWS_REGION
// const awsBucket = process.env.AWS_BUCKET_NAME
// AWS.config.update({region: awsRegion})
// const s3: S3 = new AWS.S3()
// const blockStore: BlockStore = awsBlockStore({ cache, s3, bucket: awsBucket })
const versionStore = await versionStoreFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
});
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
const tx = await g.tx();
const bible = await tx.addV(ObjectTypes.ROOT).next();
let book, book_id;
let chapter, chapter_id;
let verse, verse_id;
for (const line of lines) {
const entry = JSON.parse(line);
if (book === undefined || book_id !== entry.book_id) {
book = await tx
.addV(ObjectTypes.BOOK)
.property(KeyTypes.ID, entry.book_id, PropTypes.ANY)
.property(KeyTypes.NAME, entry.book_name, PropTypes.ANY)
.next();
book_id = entry.book_id;
await tx.addE(RlshpTypes.book).from(bible).to(book).next();
}
if (chapter === undefined || chapter_id !== entry.chapter) {
chapter = await tx
.addV(ObjectTypes.CHAPTER)
.property(KeyTypes.ID, entry.chapter, PropTypes.ANY)
.next();
chapter_id = entry.chapter;
await tx.addE(RlshpTypes.chapter).from(book).to(chapter).next();
}
if (verse === undefined || verse_id !== entry.verse) {
verse = await tx
.addV(ObjectTypes.VERSE)
.property(KeyTypes.ID, entry.verse, PropTypes.ANY)
.property(KeyTypes.TEXT, entry.text, PropTypes.ANY)
.next();
verse_id = entry.verse;
await tx.addE(RlshpTypes.verse).from(chapter).to(verse).next();
}
}
const { root, index, blocks } = await tx.commit({});
console.log(`Bible written root=${root.toString()}, storeRoot=${versionStore
.versionStoreRoot()
.toString()}`);
console.log('Version store identity', versionStore.id());
console.log('Version store log', versionStore.log());
}
async function query() {
const cache = {};
const { chunk } = chunkerFactory(1024 * 48, compute_chunks);
const linkCodec = linkCodecFactory();
const valueCodec = valueCodecFactory();
// const resolver = (cid: any) =>
// `http://192.168.1.205:8080/ipfs/${cid.toString()}`
// const blockStore = httpBlockStore({ cache, resolver })
// const awsRegion = process.env.AWS_REGION
// const awsBucket = process.env.AWS_BUCKET_NAME
// AWS.config.update({region: awsRegion})
// const s3: S3 = new AWS.S3()
// const blockStore: BlockStore = awsBlockStore({ cache, s3, bucket: awsBucket })
const ipfs = ipfsApi({ url: process.env.IPFS_API }); // eg. /ip4/192.168.1.231/tcp/5001
const blockStore = ipfsBlockStore({ cache, ipfs });
const versionRoot = linkCodec.parseString('bafkreiegljjns2rqb3z5mtdyvq2u6u2cvsahyez6bqsdjibo6737vrqhbi');
const versionStore = await versionStoreFactory({
readOnly: true,
versionRoot,
chunk,
linkCodec,
valueCodec,
blockStore,
});
const g = protoGremlinFactory({
chunk,
linkCodec,
valueCodec,
blockStore,
versionStore,
}).g();
const { result: r1, time: t1 } = await queryVerse(g, 0, 'Rev', 22, 21);
console.log(r1, t1, 'ms');
const { result: r2, time: t2 } = await queryVerse(g, 0, 'Gen', 1, 1);
console.log(r2, t2, 'ms');
const { result: r3, time: t3 } = await queryVerse(g, 0, '1John', 5, 5);
console.log(r3, t3, 'ms');
}
async function queryVerse(g, rootOffset, book, chapter, verse) {
var _a, e_1, _b, _c;
const vr = [];
const startTime = new Date().getTime();
try {
for (var _d = true, _e = __asyncValues(g
.V([rootOffset])
.out(RlshpTypes.book)
.has(ObjectTypes.BOOK, { keyType: KeyTypes.ID, operation: eq(book) })
.out(RlshpTypes.chapter)
.has(ObjectTypes.CHAPTER, {
keyType: KeyTypes.ID,
operation: eq(chapter),
})
.out(RlshpTypes.verse)
.has(ObjectTypes.VERSE, { keyType: KeyTypes.ID, operation: eq(verse) })
.values(KeyTypes.TEXT)
.maxResults(1)
.exec()), _f; _f = await _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const result = _c;
vr.push(result);
}
finally {
_d = true;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
}
finally { if (e_1) throw e_1.error; }
}
const endTime = new Date().getTime();
const time = endTime - startTime;
return { result: vr[0].value, time };
}
await publish();
// await query()
//# sourceMappingURL=tmp.js.map