@rep2recall/r2r-sqlite
Version:
R2r format engine for Rep2Recall
883 lines (761 loc) • 25.6 kB
text/typescript
import fs from "fs";
import SparkMD5 from "spark-md5";
import { srsMap, getNextReview, repeatReview } from "./quiz";
import QParser, { dotGetter } from "q2filter";
import uuid from "uuid/v4";
import { shuffle, chunk } from "./util";
import stringify from "fast-json-stable-stringify";
import Anki, { IMedia } from "ankisync";
import Db, { prop, primary, Table, Collection } from "liteorm";
import { R2rLocal, ICondOptions, IEntry, IPagedOutput, IRender, IProgress, fromSortedData, toSortedData, ankiMustache } from "@rep2recall/r2r-format";
import { toDate } from "valid-moment";
class DbDeck {
_id?: number;
name!: string;
}
class DbSource {
_id?: number;
h!: string;
name!: string;
created!: Date;
}
<DbTemplate>({name: "template", unique: [["front", "back", "css", "js"]]})
class DbTemplate {
_id?: number;
name!: string;
sourceId?: number;
front!: string;
back?: string;
css?: string;
js?: string;
}
class DbNote {
_id?: number;
key?: string;
name!: string;
sourceId?: number;
data!: Record<string, any>;
order!: Record<string, number>;
}
class DbMedia {
_id?: number;
h?: string;
sourceId?: number;
name!: string;
data!: ArrayBuffer;
}
class DbCard {
_id!: string;
deckId!: number;
templateId?: number;
noteId?: number;
front!: string;
back?: string;
mnemonic?: string;
srsLevel?: number;
nextReview?: Date;
tag?: string[];
created!: Date;
modified?: Date;
stat?: {
streak: { right: number; wrong: number };
};
}
export default class R2rSqlite extends R2rLocal {
public db!: Db;
public filename!: string;
public deck!: Collection<DbDeck>;
public card!: Collection<DbCard>;
public source!: Collection<DbSource>;
public template!: Collection<DbTemplate>;
public note!: Collection<DbNote>;
public media!: Collection<DbMedia>;
constructor(filename: string) {
super(filename);
}
public async build() {
this.db = await Db.connect(this.filename);
this.deck = await this.db.collection(new DbDeck());
this.source = await this.db.collection(new DbSource());
this.template = await this.db.collection(new DbTemplate());
this.note = await this.db.collection(new DbNote());
this.media = await this.db.collection(new DbMedia());
this.card = await this.db.collection(new DbCard());
const preNoteCreateOrUpdate = (entry: Partial<DbNote>) => {
if (entry.data) {
if (Array.isArray(entry.data)) {
const {data, order} = fromSortedData(entry.data as any[])
entry.data = data;
entry.order = order;
}
entry.key = SparkMD5.hash(stringify(entry.data));
}
}
this.note.on("pre-create", (p) => preNoteCreateOrUpdate(p.entry));
this.note.on("pre-update", (p) => preNoteCreateOrUpdate(p.set));
function preMediaCreateOrUpdate(entry: Partial<DbMedia>) {
if (entry.data && !entry.h) {
entry.h = SparkMD5.ArrayBuffer.hash(entry.data);
}
}
this.media.on("pre-create", (p) => preMediaCreateOrUpdate(p.entry));
this.media.on("pre-update", (p) => preMediaCreateOrUpdate(p.set));
return this;
}
public async close() {
await this.db.close();
return this;
}
public async reset() {
await Promise.all([
this.source.delete({}),
this.media.delete({}),
this.template.delete({}),
this.note.delete({}),
this.card.delete({}),
this.deck.delete({})
]);
return this;
}
public async parseCond(q: string,
options: ICondOptions<IEntry> = {}
): Promise<IPagedOutput<Partial<IEntry>>> {
if (options.sortBy === "random") {
q += " is:random";
delete options.sortBy;
}
const parser = new QParser<IEntry>(q, {
anyOf: new Set(["template", "front", "mnemonic", "deck", "tag"]),
isString: new Set(["template", "front", "back", "mnemonic", "deck", "tag"]),
isDate: new Set(["created", "modified", "nextReview"]),
transforms: {
"is:due": () => {
return { nextReview: { $lt: new Date() } }
}
},
filters: {
"is:distinct": (items: any[]) => {
const col: Record<string, any> = {};
for (const it of items) {
const k = it.key;
if (k) {
if (!col[k]) {
col[k] = it;
}
} else {
col[uuid()] = it;
}
}
return Object.values(col);
},
"is:duplicate": (items: any[]) => {
const col: Record<string, any[]> = {};
for (const it of items) {
const k = it.front;
col[k] = col[k] || [];
col[k].push(it);
}
return Object.values(col).filter((a) => a.length > 1).reduce((a, b) => [...a, ...b], []);
},
"is:random": (items: any[]) => {
return shuffle(items);
}
},
sortBy: options.sortBy ? {
key: options.sortBy,
desc: options.desc !== undefined ? options.desc : true
} : undefined
});
const fullCond = parser.getCondFull();
if (!options.fields) {
return {
data: [],
count: 0
};
} else if (options.fields === "*") {
options.fields = ["data", "source", "deck", "front" ,"js", "mnemonic", "modified",
"nextReview", "sCreated", "sH", "srsLevel", "stat", "tBack", "tFront", "tag",
"template", "back", "created", "css", "_id"];
}
const allFields = new Set(options.fields || []);
for (const f of (fullCond.fields || [])) {
allFields.add(f);
}
if (q.includes("is:distinct") || q.includes("is:duplicate")) {
allFields.add("data");
}
const select: Record<string, string[]> = {};
for (const f of allFields) {
switch (f) {
case "data":
select.note = select.note || [];
select.note.push(f, "order");
break;
case "source":
select.source = select.source || [];
select.source.push("name");
break;
case "sH":
case "sCreated":
select.source = select.source || [];
select.source.push(f.substr(1).toLocaleLowerCase());
break;
case "deck":
select.deck = select.deck || [];
select.deck.push("name");
break;
case "tFront":
case "tBack":
select.template = select.template || [];
select.template.push(f.substr(1).toLocaleLowerCase());
break;
case "template":
select.template = select.template || [];
select.template.push("name");
break;
case "css":
case "js":
select.template = select.template || [];
select.template.push(f);
break;
default:
select.card = select.card || [];
select.card.push(f as string);
}
}
let chain = this.card.chain(select.card as any[]);
delete select.card;
for (const [rName, rSelect] of Object.entries(select)) {
let on = `${rName}Id`;
switch(rName) {
case "source": on = "note.sourceId";
}
chain = chain.join<any>(
(this as any)[rName],
on,
"_id",
rSelect,
"left"
);
}
const data = (await chain.data()).map((c) => {
const {order, data} = c.note || {} as any;
const output = {
data: order && data ? toSortedData({order, data}) : undefined,
source: dotGetter(c, "source.name"),
sourceCreated: dotGetter(c, "source.created"),
sourceH: dotGetter(c, "source.h"),
deck: dotGetter(c, "deck.name"),
tFront: dotGetter(c, "template.front"),
tBack: dotGetter(c, "template.back"),
template: dotGetter(c, "template.name"),
css: dotGetter(c, "template.css"),
js: dotGetter(c, "template.js"),
front: dotGetter(c, "card.front"),
back: dotGetter(c, "card.back"),
mnemonic: dotGetter(c, "card.mnemonic"),
srsLevel: dotGetter(c, "card.srsLevel"),
nextReview: dotGetter(c, "card.nextReview"),
tag: dotGetter(c, "card.tag"),
created: dotGetter(c, "card.created"),
modified: dotGetter(c, "card.modified"),
stat: dotGetter(c, "card.stat"),
_id: dotGetter(c, "card._id")
};
return output;
});
const cards = parser.parse(data);
let endPoint: number | undefined;
if (options.limit) {
endPoint = (options.offset || 0) + options.limit;
}
return {
data: cards.slice(options.offset || 0, endPoint),
count: cards.length
};
}
public async insertMany(entries: IEntry[]): Promise<string[]> {
entries = await Promise.all(entries.map((e) => this.transformCreateOrUpdate(null, e))) as IEntry[];
const now = new Date();
const sIdMap: Record<string, number> = {};
await entries.filter((e) => e.sH).distinctBy((e) => e.sH!).mapAsync(async (el) => {
await this.source.create({
name: el.source!,
created: (typeof el.sCreated === "string" ? toDate(el.sCreated) : el.sCreated) || now,
h: el.sH!
}, true)
sIdMap[el.sH!] = (await this.source.get({ h: el.sH }, ["_id"]))!._id!;
});
const tIdMap: Record<string, number> = {};
await entries.filter((el) => el.template).distinctBy((el) => el.template!).mapAsync(async (el) => {
const key = {
front: el.tFront!,
back: el.tBack,
css: el.css,
js: el.js
};
await this.template.create({
...key,
name: el.template!,
sourceId: el.sH ? sIdMap[el.sH] : undefined
}, true);
tIdMap[el.template!] = (await this.template.get(key, ["_id"]))!._id!;
});
const nIdMap: Record<string, number> = {};
await entries.filter((el) => el.data).distinctBy((el) => {
(el as any).key = SparkMD5.hash(stringify(el.data!));
return (el as any).key;
}).mapAsync(async (el) => {
const {data, order} = fromSortedData(el.data!);
await this.note.create({
name: `${el.sH}/${el.template}/${el.data![0].value}`,
data,
order,
sourceId: el.sH ? sIdMap[el.sH] : undefined
}, true);
nIdMap[(el as any).key] = (await this.note.get({ data }, ["_id"]))!._id!;
})
const dMap: { [key: string]: number } = {};
const decks = entries.map((e) => e.deck);
const deckIds = await Promise.all(decks.map((d) => this.getOrCreateDeck(d)));
decks.forEach((d, i) => {
dMap[d] = deckIds[i];
});
const cIds: string[] = [];
entries.map((e) => {
const _id = uuid();
cIds.push(_id);
this.card.create({
_id,
front: e.front,
back: e.back,
mnemonic: e.mnemonic,
srsLevel: e.srsLevel,
nextReview: (typeof e.nextReview === "string" ? toDate(e.nextReview) : e.nextReview) || undefined,
deckId: dMap[e.deck],
noteId: nIdMap[(e as any).key],
templateId: tIdMap[e.template!],
created: now,
tag: e.tag
});
});
return cIds;
}
public async updateMany(ids: string[], u: Partial<IEntry>) {
if (ids.length > 900) {
for (const idc of chunk(ids, 900)) {
await this.updateMany(idc, u);
}
return;
}
const now = new Date();
const cs = await (await this.card.find({ _id: { $in: ids } }, ["_id", ...Object.keys(u) as any[]]))
.mapAsync(async (c) => {
const c0: any = Object.assign(c, await this.transformCreateOrUpdate(c._id!, u, now));
const c1: any = { _id: c._id! };
for (let [k, v] of Object.entries(c0)) {
switch (k) {
case "deck":
k = "deckId";
v = await this.getOrCreateDeck(v as string);
c1[k] = v;
break;
case "tFront":
case "tBack":
k = k.substr(1).toLocaleLowerCase();
case "css":
case "js":
const { templateId } = (await this.card.get({ _id: c._id! }, ["templateId"]))!;
await this.template.update({ _id: templateId }, { [k]: v });
break;
case "data":
const noteId = (await this.card.get({ _id: c._id! }, ["noteId"]))!.noteId!;
const n = await this.note.get({ key: noteId }, ["order", "data"]);
if (n) {
const { order, data } = n;
for (const { key, value } of v as any[]) {
if (!order![key]) {
order![key] = Math.max(...Object.values(order!)) + 1;
}
data![key] = value;
}
await this.note.update({ key: noteId }, { order, data });
} else {
const order: Record<string, number> = {};
const data: Record<string, any> = {};
for (const { key, value } of v as any[]) {
if (!order[key]) {
order[key] = Math.max(-1, ...Object.values(order)) + 1;
}
data[key] = value;
}
const key = this.getNoteKey(data)
const name = `${key}/${Object.values(data)[0]}`;
await this.note.create({ key, name, order, data });
c1.noteId = key;
}
break;
default:
c1[k] = v;
}
}
return c1;
});
for (const c of cs) {
if (Object.keys(c).length > 1) {
await this.card.update({ _id: c._id }, c);
}
}
}
public async addTags(ids: string[], tags: string[]) {
if (ids.length > 900) {
for (const idc of chunk(ids, 900)) {
await this.addTags(idc, tags);
}
return;
}
const now = new Date();
await Promise.all((await this.card.find({ _id: { $in: ids } }, ["_id", "tag"])).map((c) => {
c.modified = now;
c.tag = c.tag || [];
for (const t of tags) {
if (!c.tag.includes(t)) {
c.tag.push(t);
}
}
return this.card.update({ _id: c._id! }, c);
}));
}
public async removeTags(ids: string[], tags: string[]) {
if (ids.length > 900) {
for (const idc of chunk(ids, 900)) {
await this.removeTags(idc, tags);
}
return;
}
const now = new Date();
await Promise.all((await this.card.find({ _id: { $in: ids } }, ["_id", "tag"])).map((c) => {
c.modified = now;
const newTags: string[] = [];
for (const t of (c.tag || [])) {
if (!tags.includes(t)) {
newTags.push(t);
}
}
c.tag = newTags;
return this.card.update({ _id: c._id! }, c);
}));
}
public async deleteMany(ids: string[]) {
if (ids.length > 900) {
for (const idc of chunk(ids, 900)) {
await this.deleteMany(idc);
}
return;
}
await this.card.delete({ _id: { $in: ids } });
}
public async render(cardId: string) {
const r = await this.parseCond(`_id=${cardId}`, {
limit: 1,
fields: ["front", "back", "mnemonic", "tFront", "tBack", "data", "css", "js"]
});
const c = r.data[0] as IRender;
const { tFront, tBack, data } = c;
if (/\n/.test(c.front || "")) {
c.front = ankiMustache(tFront || "", data);
}
if (c.back && /\n/.test(c.back)) {
c.back = ankiMustache(tBack || "", data, c.front);
}
return c;
}
protected async updateSrsLevel(dSrsLevel: number, cardId: string) {
const card = await this.card.get({ _id: cardId }, ["srsLevel", "stat"]);
if (!card) {
return;
}
card.srsLevel = card.srsLevel || 0;
card.stat = card.stat || {
streak: {
right: 0,
wrong: 0
}
};
card.stat.streak = card.stat.streak || {
right: 0,
wrong: 0
}
if (dSrsLevel > 0) {
card.stat.streak.right = (card.stat.streak.right || 0) + 1;
} else if (dSrsLevel < 0) {
card.stat.streak.wrong = (card.stat.streak.wrong || 0) + 1;
}
card.srsLevel += dSrsLevel;
if (card.srsLevel >= srsMap.length) {
card.srsLevel = srsMap.length - 1;
}
if (card.srsLevel < 0) {
card.srsLevel = 0;
}
if (dSrsLevel > 0) {
card.nextReview = getNextReview(card.srsLevel);
} else {
card.nextReview = repeatReview();
}
const { srsLevel, stat, nextReview } = card;
await this.updateMany([cardId], { srsLevel, stat, nextReview });
}
protected async transformCreateOrUpdate(
cardId: string | null,
u: Partial<IEntry>,
timestamp: Date = new Date()
): Promise<Partial<IEntry>> {
let data: { key: string, value: any }[] | null = null;
let front: string = "";
if (!cardId) {
u.created = timestamp;
} else {
u.modified = timestamp;
}
if (u.front && u.front.startsWith("@template\n")) {
if (!data) {
if (cardId) {
data = await this.getData(cardId);
} else {
data = u.data || [];
}
}
u.tFront = u.front.substr("@template\n".length);
}
if (u.tFront) {
front = ankiMustache(u.tFront, data || []);
u.front = "@md5\n" + SparkMD5.hash(front);
}
if (u.back && u.back.startsWith("@template\n")) {
if (!data) {
if (cardId) {
data = await this.getData(cardId);
} else {
data = u.data || [];
}
}
u.tBack = (u.back || "").substr("@template\n".length);
if (!front && cardId) {
front = await this.getFront(cardId);
}
}
if (u.tBack) {
const back = ankiMustache(u.tBack, data || [], front);
u.back = "@md5\n" + SparkMD5.hash(back);
}
return u;
}
protected async getOrCreateDeck(name: string): Promise<number> {
try {
return await this.deck.create({ name });
} catch (e) {
return (await this.deck.get({ name }, ["_id"]))!._id!;
}
}
protected async getData(cardId: string): Promise<{ key: string, value: any }[]> {
const output: { key: string, value: any }[] = [];
const c = await this.card.get({ _id: cardId }, ["noteId"]);
if (c && c.noteId) {
const n = await this.note.get({ key: c.noteId }, ["data", "order"]);
if (n) {
for (const [k, v] of Object.entries(n.data!)) {
output[n.order![k]] = {
key: k,
value: v
};
}
}
}
return output;
}
protected async getFront(cardId: string): Promise<string> {
const c = await this.card.get({ _id: cardId }, ["front", "templateId"]);
if (c && c.front) {
if (c.front.startsWith("@md5\n") && c.templateId) {
const t = await this.template.get({ name: c.templateId }, ["front"]);
if (t) {
const data = await this.getData(cardId);
return ankiMustache(t.front!, data || []);
}
}
return c.front;
}
return "";
}
public async fromR2r(r2r: R2rLocal, options?: { filename?: string, callback?: (p: IProgress) => void }) {
const filename = options ? options.filename : undefined;
const callback = options ? options.callback : undefined;
if (callback) callback({ text: "Reading R2r file" });
const data = fs.readFileSync(r2r.filename);
const sourceH = SparkMD5.ArrayBuffer.hash(data);
const now = new Date();
let sourceId: number;
try {
sourceId = await this.source.create({
name: filename || r2r.filename,
h: sourceH,
created: now
});
} catch (e) {
if (callback) callback({ text: "Duplicated resource" });
return;
}
await (await r2r.allMedia()).mapAsync((m) => {
return this.media.create({
name: m.name!,
data: m.data!,
sourceId
}, true);
});
const cs = await r2r.parseCond("", {
fields: "*"
});
await r2r.insertMany(cs.data as IEntry[]);
}
public async export(r2r: R2rLocal, q: string = "",
options?: { callback?: (p: IProgress) => void }
) {
const callback = options ? options.callback : undefined;
let current = 1;
const ms = await this.media.find({});
for (const m of ms) {
if (callback) callback({text: "Inserting media", current, max: ms.length});
try {
await r2r.createMedia(m as IMedia);
} catch(e) {}
current++;
}
if (callback) callback({text: "Parsing q"})
const cs = await this.parseCond(q, {
fields: "*"
});
current = 1;
for (const c of chunk(cs.data as IEntry[], 1000)) {
if (callback) callback({text: "Inserting cards", current, max: cs.count});
await r2r.insertMany(c);
current += 1000;
}
await r2r.close();
}
public async getMedia(h: string): Promise<IMedia | null> {
const m = await this.media.get({ h }) as IMedia;
return m || null;
}
public async allMedia() {
return await this.media.find({}) as IMedia[];
}
public async createMedia(m: {name: string, data: ArrayBuffer}) {
const h = SparkMD5.ArrayBuffer.hash(m.data);
await this.media.create({...m, h});
return h;
}
public async deleteMedia(h: string) {
await this.media.delete({h});
return true;
}
public async fromAnki(anki: Anki, options?: { filename?: string, callback?: (p: IProgress) => void }) {
const filename = options ? options.filename : undefined;
const callback = options ? options.callback : undefined;
if (callback) callback({ text: "Reading Anki file" });
const data = fs.readFileSync(anki.filePath);
const now = new Date();
let sourceId: number;
const sourceH = SparkMD5.ArrayBuffer.hash(data);
try {
sourceId = await this.source.create({
name: filename || anki.filePath,
h: sourceH,
created: now
});
} catch (e) {
if (callback) callback({ text: "Duplicated Anki resource" });
return;
}
let current: number;
let max: number;
const media = await anki.apkg.tables.media.all();
current = 0;
max = media.length;
await media.mapAsync(async (el) => {
if (callback) callback({ text: "Inserting media", current, max });
await this.media.create({
h: el.h,
name: el.name,
data: el.data,
sourceId
}, true);
current++;
});
const card = await anki.apkg.tables.cards.all();
const dIdMap: Record<string, number> = {};
const tIdMap: Record<string, number> = {};
const nIdMap: Record<string, number> = {};
current = 1;
max = card.length;
for (const c of chunk(card, 1000)) {
if (callback) callback({ text: "Inserting cards", current, max });
await c.mapAsync(async (el) => {
if (!Object.keys(dIdMap).includes(el.deck.name)) {
const name = el.deck.name;
await this.deck.create({ name }, true);
dIdMap[name] = (await this.deck.get({ name }, ["_id"]))!._id!;
}
const t = {
name: `${sourceH}/${el.note.model.name}/${el.template.name}`,
front: el.template.qfmt,
back: el.template.afmt,
css: el.note.model.css
};
const templateKey = this.getTemplateKey(t);
if (!Object.keys(tIdMap).includes(templateKey)) {
await this.template.create({
...t,
sourceId
}, true);
tIdMap[templateKey] = (await this.template.get(t, ["_id"]))!._id!;
}
const data: Record<string, string> = {};
const order: Record<string, number> = {};
el.template.model.flds.forEach((k, i) => {
data[k] = el.note.flds[i];
order[k] = i;
});
const key = this.getNoteKey(data);
if (!Object.keys(nIdMap).includes(key)) {
await this.note.create({
key,
name: `${sourceH}/${el.note.model.name}/${el.template.name}/${el.note.flds[0]}`,
data,
order,
sourceId
}, true);
nIdMap[key] = (await this.note.get({ key }, ["_id"]))!._id!;
}
const front = ankiMustache(el.template.qfmt, data);
const back = ankiMustache(el.template.afmt, data, front);
await this.card.create({
_id: uuid(),
deckId: dIdMap[el.deck.name],
templateId: tIdMap[templateKey],
noteId: nIdMap[key],
front: `\n${SparkMD5.hash(front)}`,
back: `\n${SparkMD5.hash(back)}`,
created: now,
tag: el.note.tags
}, true);
});
current += 1000;
};
}
}