@nymphjs/driver-sqlite3
Version:
Nymph.js - SQLite3 DB Driver
1,524 lines (1,462 loc) • 115 kB
text/typescript
import SQLite3 from 'better-sqlite3';
import type {
SearchTerm,
SearchOrTerm,
SearchNotTerm,
SearchSeriesTerm,
} from '@sciactive/tokenizer';
import {
NymphDriver,
type EntityConstructor,
type EntityData,
type EntityObjectType,
type EntityInterface,
type EntityInstanceType,
type SerializedEntityData,
type FormattedSelector,
type Options,
type Selector,
EntityUniqueConstraintError,
InvalidParametersError,
NotConfiguredError,
QueryFailedError,
UnableToConnectError,
xor,
} from '@nymphjs/nymph';
import { makeTableSuffix } from '@nymphjs/guid';
import {
SQLite3DriverConfig,
SQLite3DriverConfigDefaults as defaults,
} from './conf/index.js';
class InternalStore {
public link: SQLite3.Database;
public linkWrite?: SQLite3.Database;
public connected: boolean = false;
public transactionsStarted = 0;
constructor(link: SQLite3.Database) {
this.link = link;
}
}
/**
* The SQLite3 Nymph database driver.
*/
export default class SQLite3Driver extends NymphDriver {
public config: SQLite3DriverConfig;
protected prefix: string;
// @ts-ignore: this is assigned in connect(), which is called by the constructor.
protected store: InternalStore;
static escape(input: string) {
if (input.indexOf('\x00') !== -1) {
throw new InvalidParametersError(
'SQLite3 identifiers (like entity ETYPE) cannot contain null characters.',
);
}
return '"' + input.replace(/"/g, () => '""') + '"';
}
static escapeValue(input: string) {
return "'" + input.replace(/'/g, () => "''") + "'";
}
constructor(config: Partial<SQLite3DriverConfig>, store?: InternalStore) {
super();
this.config = { ...defaults, ...config };
if (this.config.filename === ':memory:') {
this.config.explicitWrite = true;
}
this.prefix = this.config.prefix;
if (store) {
this.store = store;
} else {
this.connect();
}
}
/**
* This is used internally by Nymph. Don't call it yourself.
*
* @returns A clone of this instance.
*/
public clone() {
return new SQLite3Driver(this.config, this.store);
}
/**
* Connect to the SQLite3 database.
*
* @returns Whether this instance is connected to a SQLite3 database.
*/
public connect() {
if (this.store && this.store.connected) {
return Promise.resolve(true);
}
// Connecting
this._connect(false);
return Promise.resolve(this.store.connected);
}
private _connect(write: boolean) {
const { filename, fileMustExist, timeout, explicitWrite, wal, verbose } =
this.config;
try {
const setOptions = (link: SQLite3.Database) => {
// Set database and connection options.
if (wal) {
link.pragma('journal_mode = WAL;');
}
link.pragma('encoding = "UTF-8";');
link.pragma('foreign_keys = 1;');
link.pragma('case_sensitive_like = 1;');
for (let pragma of this.config.pragmas) {
link.pragma(pragma);
}
// Create the preg_match and regexp functions.
link.function('regexp', { deterministic: true }, ((
pattern: string,
subject: string,
) => (this.posixRegexMatch(pattern, subject) ? 1 : 0)) as (
...params: any[]
) => any);
};
let link: SQLite3.Database;
try {
link = new SQLite3(filename, {
readonly: !explicitWrite && !write,
fileMustExist,
timeout,
verbose,
});
} catch (e: any) {
if (
e.code === 'SQLITE_CANTOPEN' &&
!explicitWrite &&
!write &&
!this.config.fileMustExist
) {
// This happens when the file doesn't exist and we attempt to open it
// readonly.
// First open it in write mode.
const writeLink = new SQLite3(filename, {
readonly: false,
fileMustExist,
timeout,
verbose,
});
setOptions(writeLink);
writeLink.close();
// Now open in readonly.
link = new SQLite3(filename, {
readonly: true,
fileMustExist,
timeout,
verbose,
});
} else {
throw e;
}
}
if (!this.store) {
if (write) {
throw new Error(
'Tried to open in write without opening in read first.',
);
}
this.store = new InternalStore(link);
} else if (write) {
this.store.linkWrite = link;
} else {
this.store.link = link;
}
this.store.connected = true;
setOptions(link);
} catch (e: any) {
if (this.store) {
this.store.connected = false;
}
if (filename === ':memory:') {
throw new NotConfiguredError(
"It seems the config hasn't been set up correctly. Could not connect: " +
e?.message,
);
} else {
throw new UnableToConnectError('Could not connect: ' + e?.message);
}
}
}
/**
* Disconnect from the SQLite3 database.
*
* @returns Whether this instance is connected to a SQLite3 database.
*/
public async disconnect() {
if (this.store.connected) {
if (this.store.linkWrite && !this.config.explicitWrite) {
this.store.linkWrite.exec('PRAGMA optimize;');
this.store.linkWrite.close();
this.store.linkWrite = undefined;
}
if (this.config.explicitWrite) {
this.store.link.exec('PRAGMA optimize;');
}
this.store.link.close();
this.store.transactionsStarted = 0;
this.store.connected = false;
}
return this.store.connected;
}
public async inTransaction() {
return this.store.transactionsStarted > 0;
}
/**
* Check connection status.
*
* @returns Whether this instance is connected to a SQLite3 database.
*/
public isConnected() {
return this.store.connected;
}
private createEntitiesTable(etype: string) {
// Create the entity table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("guid" CHARACTER(24) PRIMARY KEY, "tags" TEXT, "cdate" REAL NOT NULL, "mdate" REAL NOT NULL, "user" CHARACTER(24), "group" CHARACTER(24), "acUser" INT(1), "acGroup" INT(1), "acOther" INT(1), "acRead" TEXT, "acWrite" TEXT, "acFull" TEXT);`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_cdate`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("cdate");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_mdate`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("mdate");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_tags`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("tags");`,
);
this.createEntitiesTilmeldIndexes(etype);
}
private addTilmeldColumnsAndIndexes(etype: string) {
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "user" CHARACTER(24);`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "group" CHARACTER(24);`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acUser" INT(1);`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acGroup" INT(1);`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acOther" INT(1);`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acRead" TEXT;`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acWrite" TEXT;`,
);
this.queryRun(
`ALTER TABLE ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ADD COLUMN "acFull" TEXT;`,
);
this.createEntitiesTilmeldIndexes(etype);
}
private createEntitiesTilmeldIndexes(etype: string) {
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_user_acUser`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("user", "acUser");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_group_acGroup`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("group", "acGroup");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acUser`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acUser");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acGroup`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acGroup");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acOther`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acOther");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acRead`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acRead");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acWrite`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acWrite");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}_id_acFull`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("acFull");`,
);
}
private createDataTable(etype: string) {
// Create the data table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "value" CHARACTER(1) NOT NULL, "json" BLOB, "string" TEXT, "number" REAL, "truthy" INTEGER, PRIMARY KEY("guid", "name"));`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_guid`,
)} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("guid");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_guid_name`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("guid", "name");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_name`,
)} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("name");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_name_string`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("name", "string");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_name_number`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("name", "number");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_guid_name_number`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("guid", "name", "number");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_name_truthy`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("name", "truthy");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_guid_name_truthy`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("guid", "name", "truthy");`,
);
}
private createReferencesTable(etype: string) {
// Create the references table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "reference" CHARACTER(24) NOT NULL, PRIMARY KEY("guid", "name", "reference"));`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_guid`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("guid");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_name`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("name");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_name_reference`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("name", "reference");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_reference`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("reference");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_guid_name`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("guid", "name");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_guid_name_reference`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("guid", "name", "reference");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_reference_name_guid`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("reference", "name", "guid");`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_reference_guid_name`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("reference", "guid", "name");`,
);
}
private createTokensTable(etype: string) {
// Create the tokens table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}`,
)} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "token" INTEGER NOT NULL, "position" INTEGER NOT NULL, "stem" INTEGER NOT NULL, PRIMARY KEY("guid", "name", "token", "position"));`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}_id_name_token`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}`,
)} ("name", "token");`,
);
}
private createUniquesTable(etype: string) {
// Create the unique strings table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}uniques_${etype}`,
)} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} ("guid") ON DELETE CASCADE, "unique" TEXT NOT NULL UNIQUE, PRIMARY KEY("guid", "unique"));`,
);
}
/**
* Create entity tables in the database.
*
* @param etype The entity type to create a table for. If this is blank, the default tables are created.
*/
private createTables(etype: string | null = null) {
this.startTransaction('nymph-tablecreation');
try {
if (etype != null) {
this.createEntitiesTable(etype);
this.createDataTable(etype);
this.createReferencesTable(etype);
this.createTokensTable(etype);
this.createUniquesTable(etype);
} else {
// Create the UID table.
this.queryRun(
`CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}uids`,
)} ("name" TEXT PRIMARY KEY NOT NULL, "cur_uid" INTEGER NOT NULL);`,
);
}
} catch (e: any) {
this.rollback('nymph-tablecreation');
throw e;
}
this.commit('nymph-tablecreation');
return true;
}
private query<T extends () => any>(
runQuery: T,
query: string,
etypes: string[] = [],
): ReturnType<T> {
try {
this.nymph.config.debugInfo('sqlite3:query', query);
return runQuery();
} catch (e: any) {
const errorCode = e?.code;
const errorMsg = e?.message;
if (
errorCode === 'SQLITE_ERROR' &&
errorMsg.match(/^no such table: /) &&
this.createTables()
) {
for (let etype of etypes) {
this.createTables(etype);
}
try {
return runQuery();
} catch (e2: any) {
throw new QueryFailedError(
'Query failed: ' + e2?.code + ' - ' + e2?.message,
query,
e2?.code,
);
}
} else if (
errorCode === 'SQLITE_CONSTRAINT_UNIQUE' &&
errorMsg.match(/^UNIQUE constraint failed: /)
) {
throw new EntityUniqueConstraintError(`Unique constraint violation.`);
} else {
throw new QueryFailedError(
'Query failed: ' + e?.code + ' - ' + e?.message,
query,
e?.code,
);
}
}
}
private queryArray(
query: string,
{
etypes = [],
params = {},
}: { etypes?: string[]; params?: { [k: string]: any } } = {},
) {
return this.query(
() =>
(this.store.linkWrite || this.store.link)
.prepare(query)
.iterate(params),
`${query} -- ${JSON.stringify(params)}`,
etypes,
);
}
private queryGet(
query: string,
{
etypes = [],
params = {},
}: { etypes?: string[]; params?: { [k: string]: any } } = {},
) {
return this.query(
() =>
(this.store.linkWrite || this.store.link).prepare(query).get(params),
`${query} -- ${JSON.stringify(params)}`,
etypes,
);
}
private queryRun(
query: string,
{
etypes = [],
params = {},
}: { etypes?: string[]; params?: { [k: string]: any } } = {},
) {
return this.query(
() =>
(this.store.linkWrite || this.store.link).prepare(query).run(params),
`${query} -- ${JSON.stringify(params)}`,
etypes,
);
}
public async commit(name: string) {
if (name == null || typeof name !== 'string' || name.length === 0) {
throw new InvalidParametersError(
'Transaction commit attempted without a name.',
);
}
if (this.store.transactionsStarted === 0) {
return true;
}
this.queryRun(`RELEASE SAVEPOINT ${SQLite3Driver.escape(name)};`);
this.store.transactionsStarted--;
if (
this.store.transactionsStarted === 0 &&
this.store.linkWrite &&
!this.config.explicitWrite
) {
this.store.linkWrite.exec('PRAGMA optimize;');
this.store.linkWrite.close();
this.store.linkWrite = undefined;
}
return true;
}
public async deleteEntityByID(
guid: string,
className?: EntityConstructor | string | null,
) {
let EntityClass: EntityConstructor;
if (typeof className === 'string' || className == null) {
const GetEntityClass = this.nymph.getEntityClass(className ?? 'Entity');
EntityClass = GetEntityClass;
} else {
EntityClass = className;
}
const etype = EntityClass.ETYPE;
await this.startTransaction('nymph-delete');
try {
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} WHERE "guid"=@guid;`,
{
etypes: [etype],
params: {
guid,
},
},
);
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} WHERE "guid"=@guid;`,
{
etypes: [etype],
params: {
guid,
},
},
);
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} WHERE "guid"=@guid;`,
{
etypes: [etype],
params: {
guid,
},
},
);
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}`,
)} WHERE "guid"=@guid;`,
{
etypes: [etype],
params: {
guid,
},
},
);
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}uniques_${etype}`,
)} WHERE "guid"=@guid;`,
{
etypes: [etype],
params: {
guid,
},
},
);
} catch (e: any) {
this.nymph.config.debugError('sqlite3', `Delete entity error: "${e}"`);
await this.rollback('nymph-delete');
throw e;
}
await this.commit('nymph-delete');
// Remove any cached versions of this entity.
if (this.nymph.config.cache) {
this.cleanCache(guid);
}
return true;
}
public async deleteUID(name: string) {
if (!name) {
throw new InvalidParametersError('Name not given for UID');
}
await this.startTransaction('nymph-delete-uid');
this.queryRun(
`DELETE FROM ${SQLite3Driver.escape(
`${this.prefix}uids`,
)} WHERE "name"=@name;`,
{
params: {
name,
},
},
);
await this.commit('nymph-delete-uid');
return true;
}
public async getIndexes(etype: string) {
const indexes: {
scope: 'data' | 'references' | 'tokens';
name: string;
property: string;
}[] = [];
for (let [scope, suffix] of [
['data', '_json'],
['references', '_reference_guid'],
['tokens', '_token_position_stem'],
] as (
| ['data', '_json']
| ['references', '_reference_guid']
| ['tokens', '_token_position_stem']
)[]) {
const indexDefinitions: IterableIterator<any> = this.queryArray(
`SELECT "name", "sql" FROM "sqlite_master" WHERE "type"='index' AND "name" LIKE @pattern;`,
{
params: {
pattern: `${this.prefix}${scope}_${etype}_id_custom_%${suffix}`,
},
},
);
for (const indexDefinition of indexDefinitions) {
indexes.push({
scope,
name: (indexDefinition.name as string).substring(
`${this.prefix}${scope}_${etype}_id_custom_`.length,
indexDefinition.name.length - suffix.length,
),
property:
((indexDefinition.sql as string).match(
/WHERE\s+"name"\s*=\s*'(.*)'/,
) ?? [])[1] ?? '',
});
}
}
return indexes;
}
public async addIndex(
etype: string,
definition: {
scope: 'data' | 'references' | 'tokens';
name: string;
property: string;
},
) {
this.checkIndexName(definition.name);
await this.deleteIndex(etype, definition.scope, definition.name);
if (definition.scope === 'data') {
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${definition.name}_json`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("json") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${definition.name}_string`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("string") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${definition.name}_number`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("number") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${definition.name}_truthy`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} ("truthy") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
} else if (definition.scope === 'references') {
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_custom_${definition.name}_reference_guid`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}references_${etype}`,
)} ("reference", "guid") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
} else if (definition.scope === 'tokens') {
this.queryRun(
`CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}_id_custom_${definition.name}_token_position_stem`,
)} ON ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}`,
)} ("token", "position", "stem") WHERE "name"=${SQLite3Driver.escapeValue(definition.property)};`,
);
}
return true;
}
public async deleteIndex(
etype: string,
scope: 'data' | 'references' | 'tokens',
name: string,
) {
this.checkIndexName(name);
if (scope === 'data') {
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${name}_json`,
)};`,
);
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${name}_string`,
)};`,
);
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${name}_number`,
)};`,
);
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}data_${etype}_id_custom_${name}_truthy`,
)};`,
);
} else if (scope === 'references') {
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}references_${etype}_id_custom_${name}_reference_guid`,
)};`,
);
} else if (scope === 'tokens') {
this.queryRun(
`DROP INDEX IF EXISTS ${SQLite3Driver.escape(
`${this.prefix}tokens_${etype}_id_custom_${name}_token_position_stem`,
)};`,
);
}
return true;
}
public async getEtypes() {
const tables: IterableIterator<any> = this.queryArray(
"SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix;",
{
params: {
prefix: this.prefix + 'entities_' + '%',
},
},
);
const etypes: string[] = [];
for (const table of tables) {
etypes.push(table.name.substr((this.prefix + 'entities_').length));
}
return etypes;
}
public async *exportDataIterator(): AsyncGenerator<
{ type: 'comment' | 'uid' | 'entity'; content: string },
void,
false | undefined
> {
if (
yield {
type: 'comment',
content: `#nex2
# Nymph Entity Exchange v2
# http://nymph.io
#
# Generation Time: ${new Date().toLocaleString()}
`,
}
) {
return;
}
if (
yield {
type: 'comment',
content: `
#
# UIDs
#
`,
}
) {
return;
}
// Export UIDs.
let uids: IterableIterator<any> = this.queryArray(
`SELECT * FROM ${SQLite3Driver.escape(
`${this.prefix}uids`,
)} ORDER BY "name";`,
);
for (const uid of uids) {
if (yield { type: 'uid', content: `<${uid.name}>[${uid.cur_uid}]\n` }) {
return;
}
}
if (
yield {
type: 'comment',
content: `
#
# Entities
#
`,
}
) {
return;
}
// Get the etypes.
const etypes = await this.getEtypes();
for (const etype of etypes) {
// Export entities.
const dataIterator: IterableIterator<any> = this.queryArray(
`SELECT e."guid", e."tags", e."cdate", e."mdate", e."user", e."group", e."acUser", e."acGroup", e."acOther", e."acRead", e."acWrite", e."acFull", d."name", d."value", json(d."json") as "json", d."string", d."number" FROM ${SQLite3Driver.escape(
`${this.prefix}entities_${etype}`,
)} e LEFT JOIN ${SQLite3Driver.escape(
`${this.prefix}data_${etype}`,
)} d USING ("guid") ORDER BY e."guid";`,
)[Symbol.iterator]();
let datum = dataIterator.next();
while (!datum.done) {
const guid = datum.value.guid;
const tags = datum.value.tags.slice(1, -1);
const cdate = datum.value.cdate;
const mdate = datum.value.mdate;
const user = datum.value.user;
const group = datum.value.group;
const acUser = datum.value.acUser;
const acGroup = datum.value.acGroup;
const acOther = datum.value.acOther;
const acRead = datum.value.acRead?.slice(1, -1).split(',');
const acWrite = datum.value.acWrite?.slice(1, -1).split(',');
const acFull = datum.value.acFull?.slice(1, -1).split(',');
let currentEntityExport: string[] = [];
currentEntityExport.push(`{${guid}}<${etype}>[${tags}]`);
currentEntityExport.push(`\tcdate=${JSON.stringify(cdate)}`);
currentEntityExport.push(`\tmdate=${JSON.stringify(mdate)}`);
if (this.nymph.tilmeld != null) {
if (user != null) {
currentEntityExport.push(
`\tuser=${JSON.stringify(['nymph_entity_reference', user, 'User'])}`,
);
}
if (group != null) {
currentEntityExport.push(
`\tgroup=${JSON.stringify(['nymph_entity_reference', group, 'Group'])}`,
);
}
if (acUser != null) {
currentEntityExport.push(`\tacUser=${JSON.stringify(acUser)}`);
}
if (acGroup != null) {
currentEntityExport.push(`\tacGroup=${JSON.stringify(acGroup)}`);
}
if (acOther != null) {
currentEntityExport.push(`\tacOther=${JSON.stringify(acOther)}`);
}
if (acRead != null) {
currentEntityExport.push(`\tacRead=${JSON.stringify(acRead)}`);
}
if (acWrite != null) {
currentEntityExport.push(`\tacWrite=${JSON.stringify(acWrite)}`);
}
if (acFull != null) {
currentEntityExport.push(`\tacFull=${JSON.stringify(acFull)}`);
}
}
if (datum.value.name != null) {
// This do will keep going and adding the data until the
// next entity is reached. datum will end on the next entity.
do {
const value =
datum.value.value === 'N'
? JSON.stringify(datum.value.number)
: datum.value.value === 'S'
? JSON.stringify(datum.value.string)
: datum.value.value === 'J'
? datum.value.json
: datum.value.value;
currentEntityExport.push(`\t${datum.value.name}=${value}`);
datum = dataIterator.next();
} while (!datum.done && datum.value.guid === guid);
} else {
// Make sure that datum is incremented :)
datum = dataIterator.next();
}
currentEntityExport.push('');
if (yield { type: 'entity', content: currentEntityExport.join('\n') }) {
return;
}
}
}
}
/**
* Generate the SQLite3 query.
* @param options The options array.
* @param formattedSelectors The formatted selector array.
* @param etype
* @param count Used to track internal params.
* @param params Used to store internal params.
* @param subquery Whether only a subquery should be returned.
* @returns The SQL query.
*/
private makeEntityQuery(
options: Options,
formattedSelectors: FormattedSelector[],
etype: string,
count = { i: 0 },
params: { [k: string]: any } = {},
subquery = false,
tableSuffix = '',
etypes: string[] = [],
guidSelector: string | undefined = undefined,
) {
if (typeof options.class?.alterOptions === 'function') {
options = options.class.alterOptions(options);
}
const eTable = `e${tableSuffix}`;
const dTable = `d${tableSuffix}`;
const fTable = `f${tableSuffix}`;
const ieTable = `ie${tableSuffix}`;
const sTable = `s${tableSuffix}`;
const sort = options.sort === undefined ? 'cdate' : options.sort;
const queryParts = this.iterateSelectorsForQuery(
formattedSelectors,
({ key, value, typeIsOr, typeIsNot }) => {
const clauseNot = key.startsWith('!');
let curQuery = '';
for (const curValue of value) {
switch (key) {
case 'guid':
case '!guid':
for (const curGuid of curValue) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const guid = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'."guid"=@' +
guid;
params[guid] = curGuid;
}
break;
case 'tag':
case '!tag':
for (const curTag of curValue) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const tag = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'."tags" LIKE @' +
tag +
" ESCAPE '\\'";
params[tag] =
'%,' +
curTag
.replace('\\', '\\\\')
.replace('%', '\\%')
.replace('_', '\\_') +
',%';
}
break;
case 'defined':
case '!defined':
for (const curVar of curValue) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
if (
curVar === 'cdate' ||
curVar === 'mdate' ||
(this.nymph.tilmeld != null &&
(curVar === 'user' ||
curVar === 'group' ||
curVar === 'acUser' ||
curVar === 'acGroup' ||
curVar === 'acOther' ||
curVar === 'acRead' ||
curVar === 'acWrite' ||
curVar === 'acFull'))
) {
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'(' +
ieTable +
'.' +
SQLite3Driver.escape(curVar) +
' IS NOT NULL)';
} else {
const name = `param${++count.i}`;
curQuery +=
ieTable +
'."guid" ' +
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'IN (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' WHERE "name"=@' +
name +
')';
params[name] = curVar;
}
}
break;
case 'truthy':
case '!truthy':
for (const curVar of curValue) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
if (
curVar === 'cdate' ||
curVar === 'mdate' ||
(this.nymph.tilmeld != null &&
(curVar === 'user' ||
curVar === 'group' ||
curVar === 'acUser' ||
curVar === 'acGroup' ||
curVar === 'acOther' ||
curVar === 'acRead' ||
curVar === 'acWrite' ||
curVar === 'acFull'))
) {
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'(' +
ieTable +
'.' +
SQLite3Driver.escape(curVar) +
' IS NOT NULL)';
} else {
const name = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND "truthy"=1)';
params[name] = curVar;
}
}
break;
case 'equal':
case '!equal':
if (
curValue[0] === 'cdate' ||
curValue[0] === 'mdate' ||
(this.nymph.tilmeld != null &&
(curValue[0] === 'acUser' ||
curValue[0] === 'acGroup' ||
curValue[0] === 'acOther'))
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
'=@' +
value;
params[value] = Number(curValue[1]);
} else if (
this.nymph.tilmeld != null &&
(curValue[0] === 'user' || curValue[0] === 'group')
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
'=@' +
value;
params[value] = `${curValue[1]}`;
} else if (
this.nymph.tilmeld != null &&
(curValue[0] === 'acRead' ||
curValue[0] === 'acWrite' ||
curValue[0] === 'acFull')
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
'=@' +
value;
params[value] = Array.isArray(curValue[1])
? ',' + curValue[1].join(',') + ','
: '';
} else if (typeof curValue[1] === 'number') {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const name = `param${++count.i}`;
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND "number"=@' +
value +
')';
params[name] = curValue[0];
params[value] = curValue[1];
} else if (typeof curValue[1] === 'string') {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const name = `param${++count.i}`;
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND "string"=@' +
value +
')';
params[name] = curValue[0];
params[value] = curValue[1];
} else {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
let svalue: string;
if (
curValue[1] instanceof Object &&
typeof curValue[1].toReference === 'function'
) {
svalue = JSON.stringify(curValue[1].toReference());
} else {
svalue = JSON.stringify(curValue[1]);
}
const name = `param${++count.i}`;
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND "json"=jsonb(@' +
value +
'))';
params[name] = curValue[0];
params[value] = svalue;
}
break;
case 'contain':
case '!contain':
if (
curValue[0] === 'cdate' ||
curValue[0] === 'mdate' ||
(this.nymph.tilmeld != null &&
(curValue[0] === 'acUser' ||
curValue[0] === 'acGroup' ||
curValue[0] === 'acOther'))
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
'=@' +
value;
params[value] = Number(curValue[1]);
} else if (
this.nymph.tilmeld != null &&
(curValue[0] === 'user' || curValue[0] === 'group')
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
'=@' +
value;
params[value] = `${curValue[1]}`;
} else if (
this.nymph.tilmeld != null &&
(curValue[0] === 'acRead' ||
curValue[0] === 'acWrite' ||
curValue[0] === 'acFull')
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const id = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
ieTable +
'.' +
SQLite3Driver.escape(curValue[0]) +
' LIKE @' +
id +
" ESCAPE '\\'";
params[id] =
'%,' +
curValue[1]
.replace('\\', '\\\\')
.replace('%', '\\%')
.replace('_', '\\_') +
',%';
} else {
const containTableSuffix = makeTableSuffix();
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
let svalue: string;
if (
curValue[1] instanceof Object &&
typeof curValue[1].toReference === 'function'
) {
svalue = JSON.stringify(curValue[1].toReference());
} else {
svalue = JSON.stringify(curValue[1]);
}
const name = `param${++count.i}`;
const value = `param${++count.i}`;
curQuery +=
(xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'data_' + etype) +
' d' +
containTableSuffix +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND json(@' +
value +
') IN (SELECT json_quote("value") FROM json_each(d' +
containTableSuffix +
'."json")))';
params[name] = curValue[0];
params[value] = svalue;
}
break;
case 'search':
case '!search':
if (
curValue[0] === 'cdate' ||
curValue[0] === 'mdate' ||
(this.nymph.tilmeld != null &&
(curValue[0] === 'user' ||
curValue[0] === 'group' ||
curValue[0] === 'acUser' ||
curValue[0] === 'acGroup' ||
curValue[0] === 'acOther' ||
curValue[0] === 'acRead' ||
curValue[0] === 'acWrite' ||
curValue[0] === 'acFull'))
) {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
curQuery += (xor(typeIsNot, clauseNot) ? 'NOT ' : '') + '(0)';
} else {
if (curQuery) {
curQuery += typeIsOr ? ' OR ' : ' AND ';
}
const parsedFTSQuery = this.tokenizer.parseSearchQuery(
curValue[1],
);
if (!parsedFTSQuery.length) {
curQuery += (xor(typeIsNot, clauseNot) ? 'NOT ' : '') + '(0)';
} else {
const name = `param${++count.i}`;
const queryPartToken = (term: SearchTerm) => {
const value = `param${++count.i}`;
params[value] = term.token;
return (
'EXISTS (SELECT "guid" FROM ' +
SQLite3Driver.escape(this.prefix + 'tokens_' + etype) +
' WHERE "guid"=' +
ieTable +
'."guid" AND "name"=@' +
name +
' AND "token"=@' +
value +
(term.nostemmed ? ' AND "stem"=0' : '') +
')'
);
};
const queryPartSeries = (series: SearchSeriesTerm) => {
const tokenTableSuffix = makeTableSuffix();
const tokenParts = series.tokens.map((token, i) => {
const value = `param${++count.i}`;
params[value] = token.token;
return {
fromClause:
i === 0
? 'FROM ' +
SQLite3Driver.escape(