akasharender
Version:
Rendering support for generating static HTML websites or EPUB eBooks
1,224 lines • 278 kB
JavaScript
/**
*
* Copyright 2014-2025 David Herron
*
* This file is part of AkashaCMS (http://akashacms.com/).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
if (kind === "m") throw new TypeError("Private method is not writable");
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
};
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
};
var _BaseCache_instances, _BaseCache_config, _BaseCache_name, _BaseCache_dirs, _BaseCache_is_ready, _BaseCache_db, _BaseCache_dbname, _BaseCache_vfstack, _BaseCache_fExistsInDir, _AssetsCache_insertDocAssets, _AssetsCache_updateDocAssets, _PartialsCache_insertDocPartials, _PartialsCache_updateDocPartials, _LayoutsCache_insertDocLayouts, _LayoutsCache_updateDocLayouts, _DocumentsCache_insertDocDocuments, _DocumentsCache_insertLembedDocuments, _DocumentsCache_updateDocDocuments, _DocumentsCache_updateLembedDocuments, _DocumentsCache_searchSemantic, _DocumentsCache_siblingsSQL, _DocumentsCache_docsForDirname, _DocumentsCache_dirsForParentdir, _DocumentsCache_indexFilesSQL, _DocumentsCache_indexFilesSQLrenderPath, _DocumentsCache_filesForSetTimes, _DocumentsCache_docLinkData;
import FS, { promises as fsp } from 'node:fs';
import { VFStack } from './vfstack.js';
import path from 'node:path';
import util from 'node:util';
import EventEmitter from 'events';
import micromatch from 'micromatch';
import { TagGlue, TagDescriptions } from './tag-glue.js';
import { doCreateAssetsTable, doCreateDocumentsTable, doCreateLayoutsTable, doCreatePartialsTable, doCreateVecDocumentsTable, validateAsset, validateDocument, validateLayout, validatePartial, validatePathsReturnType } from './schema.js';
import SqlString from 'sqlstring-sqlite';
import Cache from 'cache';
import { lembedModelName } from '../sqdb.js';
const tglue = new TagGlue();
// tglue.init(sqdb._db);
const tdesc = new TagDescriptions();
// tdesc.init(sqdb._db);
/**
* Base class for file caches (documents, assets, layouts, partials).
* Scans directories, stores file information in SQLite database, and emits events.
*
* Events emitted:
* - 'added' (name: string, vpath: string) - Emitted when a file is successfully
* added to the cache during initial scan or update. Useful for tracking that
* all files are processed before 'ready' is emitted.
* - 'ready' (name: string) - Emitted when initial directory scan and file
* processing is complete. After this event, isReady() will return immediately.
* - 'error' (error: Error) - Emitted when an error occurs during processing.
*/
export class BaseCache extends EventEmitter {
/**
* @param config AkashaRender Configuration object
* @param name string giving the name for this cache
* @param db The PROMISED SQLITE3 AsyncDatabase instance to use
* @param dbname The database name to use
*/
constructor(config, name, dirs, db, dbname) {
super();
_BaseCache_instances.add(this);
_BaseCache_config.set(this, void 0);
_BaseCache_name.set(this, void 0);
_BaseCache_dirs.set(this, void 0);
_BaseCache_is_ready.set(this, false);
_BaseCache_db.set(this, void 0);
_BaseCache_dbname.set(this, void 0);
_BaseCache_vfstack.set(this, void 0);
this.findPathMountedSQL = new Map();
this.findByPathSQL = new Map();
this.pathsSQL = new Map();
// console.log(`BaseFileCache ${name} constructor dirs=${util.inspect(dirs)}`);
__classPrivateFieldSet(this, _BaseCache_config, config, "f");
__classPrivateFieldSet(this, _BaseCache_name, name, "f");
__classPrivateFieldSet(this, _BaseCache_dirs, dirs, "f");
__classPrivateFieldSet(this, _BaseCache_is_ready, false, "f");
__classPrivateFieldSet(this, _BaseCache_db, db, "f");
if (dbname !== 'ASSETS'
&& dbname !== 'LAYOUTS'
&& dbname !== 'PARTIALS'
&& dbname !== 'DOCUMENTS') {
throw new Error(`Illegal database name, must be ASSETS, LAYOUTS, PARTIALS, or DOCUMENTS`);
}
__classPrivateFieldSet(this, _BaseCache_dbname, dbname, "f");
}
get config() { return __classPrivateFieldGet(this, _BaseCache_config, "f"); }
get name() { return __classPrivateFieldGet(this, _BaseCache_name, "f"); }
get dirs() { return __classPrivateFieldGet(this, _BaseCache_dirs, "f"); }
get db() { return __classPrivateFieldGet(this, _BaseCache_db, "f"); }
get dbname() { return __classPrivateFieldGet(this, _BaseCache_dbname, "f"); }
get quotedDBName() {
return SqlString.escape(__classPrivateFieldGet(this, _BaseCache_dbname, "f"));
}
async close() {
this.removeAllListeners('changed');
this.removeAllListeners('added');
this.removeAllListeners('unlinked');
this.removeAllListeners('ready');
try {
await __classPrivateFieldGet(this, _BaseCache_db, "f").close();
}
catch (err) {
// console.warn(`${this.name} error on close ${err.message}`);
}
}
/**
* Scan the directory stack and populate the database.
*/
async setup() {
__classPrivateFieldSet(this, _BaseCache_is_ready, false, "f");
__classPrivateFieldSet(this, _BaseCache_vfstack, new VFStack(this.name, this.dirs), "f");
await __classPrivateFieldGet(this, _BaseCache_vfstack, "f").scan();
for (const vpathData of __classPrivateFieldGet(this, _BaseCache_vfstack, "f")) {
if (!this.ignoreFile(vpathData)) {
try {
this.gatherInfoData(vpathData);
await this.insertDocToDB(vpathData);
await this.config.hookFileAdded(this.name, vpathData);
// Emit 'added' event to track when files are processed
// This helps verify that all files are added before 'ready' is emitted
this.emit('added', this.name, vpathData.vpath);
}
catch (err) {
console.error(`Error gathering info for ${vpathData.vpath}: ${err.message}`);
}
}
}
__classPrivateFieldSet(this, _BaseCache_is_ready, true, "f");
this.emit('ready', this.name);
}
/**
* Validate an item, which is expected to be
* a row from database query results, using
* one of the validator functions in schema.ts.
*
* This function must be subclassed to
* function correctly.
*
* @param row
*/
validateRow(row) {
throw new Error(`validateRow must be subclassed`);
}
/**
* Validate an array, which is expected to be
* database query results, using one of the
* validator functions in schema.ts.
*
* This function must be subclassed to
* function correctly.
*
* @param row
*/
validateRows(rows) {
throw new Error(`validateRows must be subclassed`);
}
/**
* Convert fields from the database
* representation to the form required
* for execution.
*
* The database cannot stores JSON fields
* as an object structure, but as a serialied
* JSON string. Inside AkashaCMS code that
* object must be an object rather than
* a string.
*
* The object passed as "row" should already
* have been validated using validateRow.
*
* @param row
*/
cvtRowToObj(row) {
if (typeof row.info === 'string') {
row.info = JSON.parse(row.info);
}
return row;
}
async sqlFormat(fname, params) {
const sql = SqlString.format(await fsp.readFile(fname), params);
return sql;
}
/**
* Find an info object based on vpath and mounted.
*
* @param vpath
* @param mounted
* @returns
*/
async findPathMounted(vpath, mounted) {
let sql = this.findPathMountedSQL.get(this.dbname);
if (!sql) {
sql = await this.sqlFormat(path.join(import.meta.dirname, 'sql', 'find-path-mounted.sql'), [this.dbname]);
this.findPathMountedSQL.set(this.dbname, sql);
}
const found = await this.db.all(sql, {
$vpath: vpath,
$mounted: mounted
});
const mapped = new Array();
for (const item of found) {
if (typeof item.vpath === 'string'
&& typeof item.mounted === 'string') {
mapped.push({
vpath: item.vpath, mounted: item.mounted
});
}
else {
throw new Error(`findPathMounted: Invalid object found in query (${vpath}, ${mounted}) results ${util.inspect(item)}`);
}
}
return mapped;
}
/**
* Find an info object by the vpath.
*
* @param vpath
* @returns
*/
async findByPath(vpath) {
const doCaching = this.config.cachingTimeout > 0;
let cacheKey;
if (!this.findByPathCache) {
this.findByPathCache
= new Cache(this.config.cachingTimeout);
}
if (doCaching) {
cacheKey = JSON.stringify({
dbname: this.quotedDBName,
vpath,
});
const cached = this.findByPathCache.get(cacheKey);
if (cached) {
return cached;
}
}
// console.log(`findByPath ${this.dao.table.quotedName} ${vpath}`);
let sql = this.findByPathSQL.get(this.dbname);
if (!sql) {
sql = await this.sqlFormat(path.join(import.meta.dirname, 'sql', 'find-by-cache.sql'), [this.dbname]);
this.findByPathSQL.set(this.dbname, sql);
}
let found;
try {
found = await this.db.all(sql, {
$vpath: vpath
});
}
catch (err) {
console.log(`db.all ${sql}`, err.stack);
throw err;
}
const mapped = this.validateRows(found);
const ret = mapped.map(item => {
return this.cvtRowToObj(item);
});
if (doCaching && cacheKey) {
this.findByPathCache.put(cacheKey, ret);
}
return ret;
}
gatherInfoData(info) {
// Placeholder which some subclasses
// are expected to override
// info.renderPath = info.vpath;
throw new Error(`gatherInfoData must be overridden`);
}
async insertDocToDB(info) {
throw new Error(`insertDocToDB must be overridden`);
}
async updateDocInDB(info) {
throw new Error(`updateDocInDB must be overridden`);
}
/**
* Allow a caller to wait until the <em>ready</em> event has
* been sent from the DirsWatcher instance. This event means the
* initial indexing has happened.
*/
async isReady() {
// If there's no directories, there won't be any files
// to load, and no need to wait
while (__classPrivateFieldGet(this, _BaseCache_dirs, "f").length > 0 && !__classPrivateFieldGet(this, _BaseCache_is_ready, "f")) {
// This does a 100ms pause
// That lets us check is_ready every 100ms
// at very little cost
// console.log(`!isReady ${this.name} ${this[_symb_dirs].length} ${this[_symb_is_ready]}`);
await new Promise((resolve, reject) => {
setTimeout(() => {
resolve(undefined);
}, 100);
});
}
return true;
}
/**
* Find the directory mount corresponding to the file.
*
* @param {*} info
* @returns
*/
fileDirMount(info) {
for (const dir of __classPrivateFieldGet(this, _BaseCache_dirs, "f")) {
if (info.mountPoint === dir.dest) {
return dir;
}
}
return undefined;
}
/**
* Should this file be ignored, based on the `ignore` field
* in the matching `dir` mount entry.
*
* @param {*} info
* @returns
*/
ignoreFile(info) {
// console.log(`ignoreFile ${info.vpath}`);
const dirMount = this.fileDirMount(info);
// console.log(`ignoreFile ${info.vpath} dirMount ${util.inspect(dirMount)}`);
let ignore = false;
if (dirMount) {
let ignores;
if (typeof dirMount.ignore === 'string') {
ignores = [dirMount.ignore];
}
else if (Array.isArray(dirMount.ignore)) {
ignores = dirMount.ignore;
}
else {
ignores = [];
}
for (const i of ignores) {
if (micromatch.isMatch(info.vpath, i))
ignore = true;
// console.log(`dirMount.ignore ${fspath} ${i} => ${ignore}`);
}
// if (ignore) console.log(`MUST ignore File ${info.vpath}`);
// console.log(`ignoreFile for ${info.vpath} ==> ${ignore}`);
return ignore;
}
else {
// no mount? that means something strange
console.error(`No dirMount found for ${info.vpath} / ${info.dirMountedOn}`);
return true;
}
}
/**
* Return simple information about each
* path in the collection. The return
* type is an array of PathsReturnType.
*
* I found two uses for this function.
* In copyAssets, the vpath and other
* simple data is used for copying items
* to the output directory.
* In render.ts, the simple fields are
* used to then call find to retrieve
* the full information.
*
* @param rootPath
* @returns
*/
async paths(rootPath) {
const fcache = this;
let rootP = rootPath?.startsWith('/')
? rootPath?.substring(1)
: rootPath;
const doCaching = this.config.cachingTimeout > 0;
let cacheKey;
if (doCaching) {
if (!this.pathsCache) {
this.pathsCache
= new Cache(this.config.cachingTimeout);
}
cacheKey = JSON.stringify({
dbname: this.quotedDBName,
rootP,
});
const cached = this.pathsCache.get(cacheKey);
if (cached) {
return cached;
}
}
let sqlRootP = this.pathsSQL.get(JSON.stringify({
dbname: this.dbname, rootP: true
}));
if (!sqlRootP) {
sqlRootP = await this.sqlFormat(path.join(import.meta.dirname, 'sql', 'paths-rootp.sql'), [this.dbname]);
this.pathsSQL.set(this.dbname, sqlRootP);
}
let sqlNoRoot = this.pathsSQL.get(JSON.stringify({
dbname: this.dbname, rootP: false
}));
if (!sqlNoRoot) {
sqlNoRoot = await this.sqlFormat(path.join(import.meta.dirname, 'sql', 'paths-no-root.sql'), [this.dbname]);
this.pathsSQL.set(this.dbname, sqlNoRoot);
}
// This is copied from the older version
// (LokiJS version) of this function. It
// seems meant to eliminate duplicates.
const vpathsSeen = new Set();
// Select the fields in PathsReturnType
const results = (typeof rootP === 'string')
? await this.db.all(sqlRootP, {
$rootP: `${rootP}%`
})
: await this.db.all(sqlNoRoot);
const result2 = new Array();
for (const item of results) {
if (fcache.ignoreFile(item)) {
continue;
}
if (vpathsSeen.has(item.vpath)) {
continue;
}
else {
vpathsSeen.add(item.vpath);
}
if (item.mime === null) {
item.mime = undefined;
}
const { error, value } = validatePathsReturnType(item);
if (error) {
console.log(`PATHS VALIDATION ${util.inspect(item)}`, error.stack);
throw error;
}
result2.push(value);
}
if (doCaching && cacheKey) {
this.pathsCache.put(cacheKey, result2);
}
return result2;
}
/**
* Find the file within the cache.
*
* @param _fpath The vpath or renderPath to look for
* @returns boolean true if found, false otherwise
*/
async find(_fpath) {
if (typeof _fpath !== 'string') {
throw new Error(`find parameter not string ${typeof _fpath}`);
}
const fpath = _fpath.startsWith('/')
? _fpath.substring(1)
: _fpath;
const fcache = this;
const result1 = await this.findByPath(fpath);
// const result1 = await this.dao.selectAll({
// or: [
// { vpath: { eq: fpath }},
// { renderPath: { eq: fpath }}
// ]
// } as Filter<T>);
// console.log(`find ${_fpath} ${fpath} ==> result1 ${util.inspect(result1)} `);
const result2 = result1.filter(item => {
return !(fcache.ignoreFile(item));
});
// for (const result of result2) {
// this.gatherInfoData(result);
// }
// console.log(`find ${_fpath} ${fpath} ==> result2 ${util.inspect(result2)} `);
let ret;
if (Array.isArray(result2) && result2.length > 0) {
ret = result2[0];
}
else if (Array.isArray(result2) && result2.length <= 0) {
ret = undefined;
}
else {
ret = result2;
}
if (ret) {
const value = this.cvtRowToObj(this.validateRow(ret));
return value;
}
else {
return undefined;
}
// PROBLEM:
// the metadata, docMetadata, baseMetadata,
// and info fields, are stored in
// the database as strings, but need
// to be unpacked into objects.
//
// Using validateRow or validateRows is
// useful, but does not convert those
// fields.
}
/**
* Fulfills the "find" operation not by
* looking in the database, but by scanning
* the filesystem using synchronous calls.
*
* NOTE: This is used in partialSync
*
* @param _fpath
* @returns
*/
findSync(_fpath) {
if (typeof _fpath !== 'string') {
throw new Error(`find parameter not string ${typeof _fpath}`);
}
const fpath = _fpath.startsWith('/')
? _fpath.substring(1)
: _fpath;
// console.log(`findSync looking for ${fpath} in ${util.inspect(this.#dirs)}`);
for (const dir of __classPrivateFieldGet(this, _BaseCache_dirs, "f")) {
if (!(dir?.dest)) {
console.warn(`findSync bad dirs in ${util.inspect(this.dirs)}`);
}
const found = __classPrivateFieldGet(this, _BaseCache_instances, "m", _BaseCache_fExistsInDir).call(this, fpath, dir);
if (found) {
// console.log(`findSync ${fpath} found`, found);
return found;
}
}
return undefined;
}
}
_BaseCache_config = new WeakMap(), _BaseCache_name = new WeakMap(), _BaseCache_dirs = new WeakMap(), _BaseCache_is_ready = new WeakMap(), _BaseCache_db = new WeakMap(), _BaseCache_dbname = new WeakMap(), _BaseCache_vfstack = new WeakMap(), _BaseCache_instances = new WeakSet(), _BaseCache_fExistsInDir = function _BaseCache_fExistsInDir(fpath, dir) {
// console.log(`#fExistsInDir ${fpath} ${util.inspect(dir)}`);
if (dir.dest === '/') {
const fspath = path.join(dir.src, fpath);
let fsexists = FS.existsSync(fspath);
if (fsexists) {
let stats = FS.statSync(fspath);
return {
vpath: fpath,
renderPath: fpath,
fspath: fspath,
mime: undefined,
mounted: dir.src,
mountPoint: dir.dest,
pathInMounted: fpath,
statsMtime: stats.mtimeMs
};
}
else {
return undefined;
}
}
let mp = dir.dest.startsWith('/')
? dir.dest.substring(1)
: dir.dest;
mp = mp.endsWith('/')
? mp
: (mp + '/');
if (fpath.startsWith(mp)) {
let pathInMounted = fpath.replace(dir.dest, '');
let fspath = path.join(dir.src, pathInMounted);
// console.log(`Checking exist for ${dir.dest} ${dir.src} ${pathInMounted} ${fspath}`);
let fsexists = FS.existsSync(fspath);
if (fsexists) {
let stats = FS.statSync(fspath);
return {
vpath: fpath,
renderPath: fpath,
fspath: fspath,
mime: undefined,
mounted: dir.src,
mountPoint: dir.dest,
pathInMounted: pathInMounted,
statsMtime: stats.mtimeMs
};
}
}
return undefined;
};
export class AssetsCache extends BaseCache {
constructor() {
super(...arguments);
_AssetsCache_insertDocAssets.set(this, void 0);
_AssetsCache_updateDocAssets.set(this, void 0);
}
validateRow(row) {
const { error, value } = validateAsset(row);
if (error) {
// console.error(`ASSET VALIDATION ERROR for`, row);
throw error;
}
else
return value;
}
validateRows(rows) {
if (!Array.isArray(rows)) {
throw new Error(`AssetsCache validateRows must be given an array`);
}
const ret = new Array();
for (const row of rows) {
ret.push(this.validateRow(row));
}
return ret;
}
cvtRowToObj(row) {
if (typeof row.info === 'string') {
row.info = JSON.parse(row.info);
}
return row;
}
gatherInfoData(info) {
if (typeof info.statsMtime === 'number') {
info.mtimeMs = info.statsMtime;
}
if (info.mime === null) {
info.mime = undefined;
}
}
async insertDocToDB(info) {
if (!__classPrivateFieldGet(this, _AssetsCache_insertDocAssets, "f")) {
__classPrivateFieldSet(this, _AssetsCache_insertDocAssets, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'insert-doc-assets.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _AssetsCache_insertDocAssets, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info)
});
}
async updateDocInDB(info) {
if (!__classPrivateFieldGet(this, _AssetsCache_updateDocAssets, "f")) {
__classPrivateFieldSet(this, _AssetsCache_updateDocAssets, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'update-doc-assets.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _AssetsCache_updateDocAssets, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info)
});
}
}
_AssetsCache_insertDocAssets = new WeakMap(), _AssetsCache_updateDocAssets = new WeakMap();
export class PartialsCache extends BaseCache {
constructor() {
super(...arguments);
_PartialsCache_insertDocPartials.set(this, void 0);
_PartialsCache_updateDocPartials.set(this, void 0);
}
validateRow(row) {
const { error, value } = validatePartial(row);
if (error)
throw error;
else
return value;
}
validateRows(rows) {
if (!Array.isArray(rows)) {
throw new Error(`PartialsCache validateRows must be given an array`);
}
const ret = new Array();
for (const row of rows) {
ret.push(this.validateRow(row));
}
return ret;
}
cvtRowToObj(row) {
if (typeof row.info === 'string') {
row.info = JSON.parse(row.info);
}
return row;
}
gatherInfoData(info) {
let renderer = this.config.findRendererPath(info.vpath);
if (typeof info.statsMtime === 'number') {
info.mtimeMs = info.statsMtime;
}
if (info.mime === null) {
info.mime = undefined;
}
if (renderer) {
info.rendererName = renderer.name;
if (renderer.parseMetadata) {
const rc = renderer.parseMetadata({
fspath: info.fspath,
content: FS.readFileSync(info.fspath, 'utf-8')
});
// docBody is the parsed body -- e.g. following the frontmatter
info.docBody = rc.body;
}
}
}
async insertDocToDB(info) {
if (!__classPrivateFieldGet(this, _PartialsCache_insertDocPartials, "f")) {
__classPrivateFieldSet(this, _PartialsCache_insertDocPartials, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'insert-doc-partials.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _PartialsCache_insertDocPartials, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$docBody: info.docBody,
$rendererName: info.rendererName
});
}
async updateDocInDB(info) {
if (!__classPrivateFieldGet(this, _PartialsCache_updateDocPartials, "f")) {
__classPrivateFieldSet(this, _PartialsCache_updateDocPartials, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'update-doc-partials.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _PartialsCache_updateDocPartials, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$docBody: info.docBody,
$rendererName: info.rendererName
});
}
}
_PartialsCache_insertDocPartials = new WeakMap(), _PartialsCache_updateDocPartials = new WeakMap();
export class LayoutsCache extends BaseCache {
constructor() {
super(...arguments);
_LayoutsCache_insertDocLayouts.set(this, void 0);
_LayoutsCache_updateDocLayouts.set(this, void 0);
}
validateRow(row) {
const { error, value } = validateLayout(row);
if (error)
throw error;
else
return value;
}
validateRows(rows) {
if (!Array.isArray(rows)) {
throw new Error(`LayoutsCache validateRows must be given an array`);
}
const ret = new Array();
for (const row of rows) {
ret.push(this.validateRow(row));
}
return ret;
}
cvtRowToObj(row) {
if (typeof row.info === 'string') {
row.info = JSON.parse(row.info);
}
return row;
}
gatherInfoData(info) {
let renderer = this.config.findRendererPath(info.vpath);
if (typeof info.statsMtime === 'number') {
info.mtimeMs = info.statsMtime;
}
if (info.mime === null) {
info.mime = undefined;
}
if (renderer) {
info.rendererName = renderer.name;
const renderPath = renderer.filePath(info.vpath);
info.rendersToHTML =
micromatch.isMatch(renderPath, '**/*.html')
|| micromatch.isMatch(renderPath, '*.html')
? true : false;
if (renderer.parseMetadata) {
const rc = renderer.parseMetadata({
fspath: info.fspath,
content: FS.readFileSync(info.fspath, 'utf-8')
});
// docBody is the parsed body -- e.g. following the frontmatter
info.docBody = rc.body;
}
}
else {
info.rendersToHTML = false;
}
}
async insertDocToDB(info) {
if (!__classPrivateFieldGet(this, _LayoutsCache_insertDocLayouts, "f")) {
__classPrivateFieldSet(this, _LayoutsCache_insertDocLayouts, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'insert-doc-layouts.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _LayoutsCache_insertDocLayouts, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$rendersToHTML: info.rendersToHTML,
$docBody: info.docBody,
$rendererName: info.rendererName
});
}
async updateDocInDB(info) {
if (!__classPrivateFieldGet(this, _LayoutsCache_updateDocLayouts, "f")) {
__classPrivateFieldSet(this, _LayoutsCache_updateDocLayouts, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'update-doc-layouts.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _LayoutsCache_updateDocLayouts, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$rendersToHTML: info.rendersToHTML,
$docBody: info.docBody,
$rendererName: info.rendererName
});
}
}
_LayoutsCache_insertDocLayouts = new WeakMap(), _LayoutsCache_updateDocLayouts = new WeakMap();
export class DocumentsCache extends BaseCache {
constructor() {
super(...arguments);
// NOTE: Certain fields are not handled
// here because they're GENERATED from
// JSON data.
//
// publicationTime
// baseMetadata
// metadata
// tags
// layout
// blogtag
//
// Those fields are not touched by
// the insert/update functions because
// SQLITE3 takes care of it.
_DocumentsCache_insertDocDocuments.set(this, void 0);
_DocumentsCache_insertLembedDocuments.set(this, void 0);
_DocumentsCache_updateDocDocuments.set(this, void 0);
_DocumentsCache_updateLembedDocuments.set(this, void 0);
_DocumentsCache_searchSemantic.set(this, void 0);
_DocumentsCache_siblingsSQL.set(this, void 0);
_DocumentsCache_docsForDirname.set(this, void 0);
_DocumentsCache_dirsForParentdir.set(this, void 0);
_DocumentsCache_indexFilesSQL.set(this, void 0);
_DocumentsCache_indexFilesSQLrenderPath.set(this, void 0);
_DocumentsCache_filesForSetTimes.set(this, void 0);
_DocumentsCache_docLinkData.set(this, void 0);
}
validateRow(row) {
const { error, value } = validateDocument(row);
if (error) {
console.error(`DOCUMENT VALIDATION ERROR for ${util.inspect(row)}`, error.stack);
throw error;
}
else
return value;
}
validateRows(rows) {
if (!Array.isArray(rows)) {
throw new Error(`DocumentsCache validateRows must be given an array`);
}
const ret = new Array();
for (const row of rows) {
ret.push(this.validateRow(row));
}
return ret;
}
cvtRowToObj(row) {
// console.log(`Documents cvtRowToObj`, row);
if (typeof row.info === 'string') {
row.info = JSON.parse(row.info);
}
if (typeof row.baseMetadata === 'string') {
row.baseMetadata
= JSON.parse(row.baseMetadata);
}
if (typeof row.docMetadata === 'string') {
row.docMetadata
= JSON.parse(row.docMetadata);
}
if (typeof row.metadata === 'string') {
row.metadata
= JSON.parse(row.metadata);
}
if (typeof row.tags === 'string') {
row.tags
= JSON.parse(row.tags);
}
return row;
}
gatherInfoData(info) {
info.renderPath = info.vpath;
info.dirname = path.dirname(info.vpath);
if (info.dirname === '.')
info.dirname = '/';
info.parentDir = path.dirname(info.dirname);
// find the mounted directory,
// get the baseMetadata
for (let dir of this.dirs) {
if (dir.src === info.mounted) {
if (dir.baseMetadata) {
info.baseMetadata = dir.baseMetadata;
}
break;
}
}
if (typeof info.statsMtime === 'number') {
info.mtimeMs = info.statsMtime;
}
if (info.mime === null) {
info.mime = undefined;
}
let renderer = this.config.findRendererPath(info.vpath);
if (renderer) {
info.rendererName = renderer.name;
info.renderPath
= renderer.filePath(info.vpath);
info.rendersToHTML =
micromatch.isMatch(info.renderPath, '**/*.html')
|| micromatch.isMatch(info.renderPath, '*.html')
? true : false;
if (renderer.parseMetadata) {
const rc = renderer.parseMetadata({
fspath: info.fspath,
content: FS.readFileSync(info.fspath, 'utf-8')
});
// docMetadata is the unmodified metadata/frontmatter
// in the document
info.docMetadata = rc.metadata;
// docContent is the unparsed original content
// including any frontmatter
info.docContent = rc.content;
// docBody is the parsed body -- e.g. following the frontmatter
info.docBody = rc.body;
// This is the computed metadata that includes data from
// several sources
info.metadata = {};
if (!info.docMetadata)
info.docMetadata = {};
// The rest of this is adapted from the old function
// HTMLRenderer.newInitMetadata
// For starters the metadata is collected from several sources.
// 1) the metadata specified in the directory mount where
// this document was found
// 2) metadata in the project configuration
// 3) the metadata in the document, as captured in docMetadata
for (let yprop in info.baseMetadata) {
// console.log(`initMetadata ${basedir} ${fpath} baseMetadata ${baseMetadata[yprop]}`);
info.metadata[yprop] = info.baseMetadata[yprop];
}
for (let yprop in this.config.metadata) {
info.metadata[yprop] = this.config.metadata[yprop];
}
let fmmcount = 0;
for (let yprop in info.docMetadata) {
info.metadata[yprop] = info.docMetadata[yprop];
fmmcount++;
}
// The rendered version of the content lands here
info.metadata.content = "";
// The document object has been useful for
// communicating the file path and other data.
info.metadata.document = {};
info.metadata.document.basedir = info.mountPoint;
info.metadata.document.relpath = info.pathInMounted;
info.metadata.document.relrender = renderer.filePath(info.pathInMounted);
info.metadata.document.path = info.vpath;
info.metadata.document.renderTo = info.renderPath;
// Ensure the <em>tags</em> field is an array
if (!(info.metadata.tags)) {
info.metadata.tags = [];
}
else if (typeof (info.metadata.tags) === 'string') {
let taglist = [];
const re = /\s*,\s*/;
info.metadata.tags.split(re).forEach(tag => {
taglist.push(tag.trim());
});
info.metadata.tags = taglist;
}
else if (!Array.isArray(info.metadata.tags)) {
throw new Error(`FORMAT ERROR - ${info.vpath} has badly formatted tags `, info.metadata.tags);
}
info.docMetadata.tags = info.metadata.tags;
// The root URL for the project
info.metadata.root_url = this.config.root_url;
// Compute the URL this document will render to
if (this.config.root_url) {
let uRootUrl = new URL(this.config.root_url, 'http://example.com');
uRootUrl.pathname = path.normalize(path.join(uRootUrl.pathname, info.metadata.document.renderTo));
info.metadata.rendered_url = uRootUrl.toString();
}
else {
info.metadata.rendered_url = info.metadata.document.renderTo;
}
// info.metadata.rendered_date = info.stats.mtime;
const parsePublDate = (date) => {
const parsed = Date.parse(date);
if (!isNaN(parsed)) {
info.metadata.publicationDate
= new Date(parsed);
// info.publicationDate = info.metadata.publicationDate;
info.publicationTime
= info.metadata.publicationDate.getTime();
}
};
if (info.docMetadata
&& typeof info.docMetadata.publDate === 'string') {
parsePublDate(info.docMetadata.publDate);
}
if (info.docMetadata
&& typeof info.docMetadata.publicationDate === 'string') {
parsePublDate(info.docMetadata.publicationDate);
}
if (!info.metadata.publicationDate) {
var dateSet = false;
if (info.docMetadata
&& info.docMetadata.publDate) {
parsePublDate(info.docMetadata.publDate);
dateSet = true;
}
if (info.docMetadata
&& typeof info.docMetadata.publicationDate === 'string') {
parsePublDate(info.docMetadata.publicationDate);
dateSet = true;
}
if (!dateSet && info.mtimeMs) {
info.metadata.publicationDate
= new Date(info.mtimeMs);
// info.publicationDate = info.metadata.publicationDate;
info.publicationTime
= info.mtimeMs;
// console.log(`${info.vpath} metadata.publicationDate ${info.metadata.publicationDate} set from stats.mtime`);
}
if (!info.metadata.publicationDate) {
info.metadata.publicationDate
= new Date();
// info.publicationDate = info.metadata.publicationDate;
info.publicationTime
= info.metadata.publicationDate.getTime();
// console.log(`${info.vpath} metadata.publicationDate ${info.metadata.publicationDate} set from current time`);
}
}
}
}
else {
info.rendersToHTML = false;
info.docMetadata = {};
info.docBody = '';
info.docContent = '';
info.rendererName = '';
info.publicationTime = 0;
}
}
async insertDocToDB(info) {
if (!__classPrivateFieldGet(this, _DocumentsCache_insertDocDocuments, "f")) {
__classPrivateFieldSet(this, _DocumentsCache_insertDocDocuments, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'insert-doc-documents.sql'), 'utf-8'), "f");
}
if (!__classPrivateFieldGet(this, _DocumentsCache_insertLembedDocuments, "f")) {
__classPrivateFieldSet(this, _DocumentsCache_insertLembedDocuments, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'insert-lembed-documents.sql'), 'utf-8'), "f");
}
// let mtime;
// if (typeof info.mtimeMs === 'number'
// || typeof info.mtimeMs === 'string'
// ) {
// mtime = new Date(info.mtimeMs).toISOString();
// }
const toInsert = {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$renderPath: info.renderPath,
$rendersToHTML: info.rendersToHTML,
$parentDir: info.parentDir,
$docMetadata: JSON.stringify(info.docMetadata),
$docContent: info.docContent,
$docBody: info.docBody,
$rendererName: info.rendererName
};
// console.log(`insert doc ${info.vpath}`, toInsert);
await this.db.run(__classPrivateFieldGet(this, _DocumentsCache_insertDocDocuments, "f"), toInsert);
// if (typeof lembedModelName === 'string') {
// console.log({
// lembedModelName,
// bodyType: typeof info.docBody
// });
// } else {
// console.log({
// typeName: typeof lembedModelName,
// bodyType: typeof info.docBody
// })
// }
// This handles computing embeddings
// for the title and body
if (typeof lembedModelName === 'string'
// && typeof info.title === 'string'
&& typeof info.docBody === 'string') {
console.log(__classPrivateFieldGet(this, _DocumentsCache_insertLembedDocuments, "f"), {
$vpath: info.vpath,
$lembedModel: lembedModelName,
// $titleEmbed: info.title,
$bodyEmbed: info.docBody
});
await this.db.run(__classPrivateFieldGet(this, _DocumentsCache_insertLembedDocuments, "f"), {
$vpath: info.vpath,
$lembedModel: lembedModelName,
// $titleEmbed: info.title,
$bodyEmbed: info.docBody
});
console.log(`vec_documents inserted ${info.vpath}`);
}
if (info.metadata) {
// console.log({
// vpath: info.vpath,
// tags: info.metadata.tags
// });
await this.addDocTagGlue(info.vpath, info.metadata.tags);
}
}
async updateDocInDB(info) {
if (!__classPrivateFieldGet(this, _DocumentsCache_updateDocDocuments, "f")) {
__classPrivateFieldSet(this, _DocumentsCache_updateDocDocuments, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'update-doc-documents.sql'), 'utf-8'), "f");
}
if (!__classPrivateFieldGet(this, _DocumentsCache_updateLembedDocuments, "f")) {
__classPrivateFieldSet(this, _DocumentsCache_updateLembedDocuments, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'update-lembed-documents.sql'), 'utf-8'), "f");
}
await this.db.run(__classPrivateFieldGet(this, _DocumentsCache_updateDocDocuments, "f"), {
$vpath: info.vpath,
$mime: info.mime,
$mounted: info.mounted,
$mountPoint: info.mountPoint,
$pathInMounted: info.pathInMounted,
$fspath: path.join(info.mounted, info.pathInMounted),
$dirname: path.dirname(info.vpath),
$mtimeMs: info.mtimeMs,
$info: JSON.stringify(info),
$renderPath: info.renderPath,
$rendersToHTML: info.rendersToHTML,
$parentDir: info.parentDir,
$docMetadata: JSON.stringify(info.docMetadata),
$docContent: info.docContent,
$docBody: info.docBody,
$rendererName: info.rendererName
});
// This handles computing embeddings
// for the title and body
if (typeof lembedModelName === 'string'
// && typeof info.title === 'string'
&& typeof info.docBody === 'string') {
await this.db.run(__classPrivateFieldGet(this, _DocumentsCache_updateLembedDocuments, "f"), {
$vpath: info.vpath,
$lembedModel: lembedModelName,
// $titleEmbed: info.title,
$bodyEmbed: info.docBody
});
}
await tglue.deleteTagGlue(info.vpath);
if (info.metadata) {
await tglue.addTagGlue(info.vpath, info.metadata.tags);
}
}
async deleteDocTagGlue(vpath) {
try {
await tglue.deleteTagGlue(vpath);
}
catch (err) {
// ignore
// This can throw an error like:
// documentsCache ERROR {
// code: 'changed',
// name: 'documents',
// vpath: '_mermaid/render3356739382.mermaid',
// error: Error: delete from 'TAGGLUE' failed: nothing changed
// ... stack trace
// }
// In such a case there is no tagGlue for the document.
// This "error" is spurious.
//
// TODO Is there another query to run that will
// not throw an error if nothing was changed?
// In other words, this could hide a legitimate
// error.
}
}
async addDocTagGlue(vpath, tags) {
if (typeof tags !== 'string'
&& !Array.isArray(tags)) {
throw new Error(`addDocTagGlue must be given a tags array, was given: ${util.inspect(tags)}`);
}
await tglue.addTagGlue(vpath, Array.isArray(tags)
? tags
: [tags]);
}
async addTagDescription(tag, description) {
return tdesc.addDesc(tag, description);
}
async getTagDescription(tag) {
return tdesc.getDesc(tag);
}
async semanticSearchDocs(searchFor) {
if (!__classPrivateFieldGet(this, _DocumentsCache_searchSemantic, "f")) {
__classPrivateFieldSet(this, _DocumentsCache_searchSemantic, await fsp.readFile(path.join(import.meta.dirname, 'sql', 'doc-search-semantic.sql'), 'utf-8'), "f");
}
const results = await this.db