@himorishige/noren-dict-reloader
Version:
Dynamic dictionary and policy hot-reloading for Noren using ETag-based updates
292 lines (291 loc) • 10.3 kB
JavaScript
export class PolicyDictReloader {
opts;
timer = null;
running = false;
backoff = 0;
policy = {};
manifest = {};
dicts = new Map();
compiled = null;
loader;
constructor(opts) {
this.opts = {
intervalMs: opts.intervalMs ?? 30_000,
maxIntervalMs: opts.maxIntervalMs ?? 5 * 60_000,
jitter: opts.jitter ?? 0.2,
headers: opts.headers ?? {},
policyUrl: opts.policyUrl,
dictManifestUrl: opts.dictManifestUrl,
onSwap: opts.onSwap,
onError: opts.onError,
compile: opts.compile,
requestTimeoutMs: opts.requestTimeoutMs,
validateUrl: opts.validateUrl,
maxConcurrent: opts.maxConcurrent,
compileOptions: opts.compileOptions,
};
// Create enhanced loader with timeout and validation
this.loader =
opts.load ??
createEnhancedLoader({
requestTimeoutMs: this.opts.requestTimeoutMs,
validateUrl: this.opts.validateUrl,
});
}
getCompiled() {
if (!this.compiled)
throw new Error('not compiled');
return this.compiled;
}
async start() {
if (this.running)
return;
this.running = true;
await this.tick(false);
}
stop() {
this.running = false;
if (this.timer) {
clearTimeout(this.timer);
this.timer = null;
}
}
async forceReload() {
await this.tick(true);
}
// Optimized internal implementation
nextDelay(ok) {
this.backoff = ok
? 0
: Math.min((this.backoff || 1) * 2, Math.floor(this.opts.maxIntervalMs / this.opts.intervalMs));
const base = this.opts.intervalMs * (this.backoff || 1);
const jitter = 1 + (Math.random() * 2 * this.opts.jitter - this.opts.jitter);
return Math.floor(base * jitter);
}
schedule(ok) {
if (!this.running)
return;
this.timer = setTimeout(() => this.tick(false).catch(() => { }), this.nextDelay(ok));
}
async tick(force) {
try {
const changed = [];
// Load policy and manifest (keep sequential for critical config)
try {
const p = await this.loader(this.opts.policyUrl, this.policy, this.opts.headers, force);
if (p.status === 200) {
this.policy = p.meta;
changed.push('policy');
}
}
catch (e) {
this.opts.onError?.(new Error(`Failed to load policy: ${e instanceof Error ? e.message : String(e)}`));
// Continue with existing policy
}
try {
const m = await this.loader(this.opts.dictManifestUrl, this.manifest, this.opts.headers, force);
if (m.status === 200) {
this.manifest = m.meta;
changed.push('manifest');
}
}
catch (e) {
this.opts.onError?.(new Error(`Failed to load manifest: ${e instanceof Error ? e.message : String(e)}`));
// Continue with existing manifest
}
// Parallel dictionary loading with error resilience
const list = parseManifestJson(this.manifest.json);
if (list.length > 0) {
const maxConcurrent = this.opts.maxConcurrent ?? 4;
const dictResults = await this.loadDictionariesWithConcurrency(list, force, maxConcurrent);
// Process successful dictionary loads
for (const result of dictResults) {
if (result.status === 'fulfilled' && result.value) {
const { id, meta } = result.value;
this.dicts.set(id, meta);
changed.push(`dict:${id}`);
}
else if (result.status === 'rejected') {
this.opts.onError?.(new Error(`Dictionary load failed: ${result.reason}`));
// Keep existing dictionary data
}
}
}
// Remove dictionaries no longer in manifest
for (const existingId of Array.from(this.dicts.keys())) {
if (!list.find((x) => x.id === existingId)) {
this.dicts.delete(existingId);
changed.push(`dict-removed:${existingId}`);
}
}
if (changed.length === 0 && this.compiled && !force) {
this.schedule(true);
return;
}
// Compile with available data (may include partial failures)
const policyRaw = this.policy.json ?? {};
const dictsRaw = Array.from(this.dicts.values()).map((v) => v.json ?? {});
const compiled = this.opts.compile(policyRaw, dictsRaw, this.opts.compileOptions);
this.compiled = compiled;
this.opts.onSwap?.(compiled, changed);
this.schedule(true);
}
catch (e) {
this.opts.onError?.(e);
this.schedule(false);
}
}
async loadDictionariesWithConcurrency(list, force, maxConcurrent) {
const semaphore = new Semaphore(maxConcurrent);
const tasks = list.map(async ({ id, url }) => {
await semaphore.acquire();
try {
const prev = this.dicts.get(id);
const d = await this.loader(url, prev, this.opts.headers, force);
if (d.status === 200) {
return { id, meta: d.meta };
}
return null;
}
finally {
semaphore.release();
}
});
return Promise.allSettled(tasks);
}
}
// ---- utility classes ----
// Simplified semaphore for concurrency control
class Semaphore {
available;
waiting = [];
constructor(maxConcurrent) {
this.available = maxConcurrent;
}
async acquire() {
if (this.available > 0) {
this.available--;
return;
}
return new Promise((resolve) => this.waiting.push(resolve));
}
release() {
const next = this.waiting.shift();
if (next)
next();
else
this.available++;
}
}
// ---- fetch helpers ----
/**
* Create an enhanced loader with timeout and URL validation
*/
function createEnhancedLoader(opts) {
return async (url, prev, headers, force) => {
// URL validation
if (opts?.validateUrl) {
try {
const parsedUrl = new URL(url);
if (!opts.validateUrl(parsedUrl)) {
throw new Error(`URL not allowed by validation: ${url}`);
}
}
catch (err) {
if (err instanceof Error && err.message.includes('not allowed')) {
throw err;
}
throw new Error(`Invalid URL: ${url}`);
}
}
return conditionalGet(url, prev, headers, force, opts?.requestTimeoutMs);
};
}
// Unified conditional GET with optional timeout support
async function conditionalGet(url, prev, headers, force, timeoutMs) {
// Prepare headers with cache control
const h = { 'cache-control': 'no-cache', ...(headers ?? {}) };
let reqUrl = url;
if (force) {
const u = new URL(url);
u.searchParams.set('_bust', String(Date.now()));
reqUrl = u.toString();
h.pragma = 'no-cache';
}
else {
if (prev?.etag)
h['if-none-match'] = prev.etag;
else if (prev?.lastModified)
h['if-modified-since'] = prev.lastModified;
}
// Setup timeout if specified
let controller;
let timeoutId;
if (timeoutMs && timeoutMs > 0) {
controller = new AbortController();
timeoutId = setTimeout(() => controller?.abort(), timeoutMs);
}
try {
const res = await fetch(reqUrl, {
method: 'GET',
headers: h,
signal: controller?.signal,
});
if (timeoutId)
clearTimeout(timeoutId);
if (res.status === 304) {
if (!prev) {
throw new Error(`304 Not Modified from ${reqUrl} without previous metadata`);
}
return { status: 304, meta: prev };
}
if (!res.ok)
throw new Error(`fetch ${reqUrl} -> ${res.status}`);
const text = await res.text();
let etag = res.headers.get('etag') ?? undefined;
const lastModified = res.headers.get('last-modified') ?? undefined;
if (!etag)
etag = `W/"sha256:${sha256Hex(text)}"`;
let json;
try {
json = JSON.parse(text);
}
catch {
json = text;
}
return { status: 200, meta: { etag, lastModified, text, json } };
}
catch (err) {
if (timeoutId)
clearTimeout(timeoutId);
if (err instanceof Error && err.name === 'AbortError') {
throw new Error(`Request timeout after ${timeoutMs}ms: ${reqUrl}`);
}
throw err;
}
}
// Optimized SHA256 hex computation with pre-compiled encoder
const enc = new TextEncoder();
async function sha256Hex(s) {
const buf = await crypto.subtle.digest('SHA-256', enc.encode(s));
return Array.from(new Uint8Array(buf), (byte) => byte.toString(16).padStart(2, '0')).join('');
}
function parseManifestJson(j) {
if (!j || typeof j !== 'object')
return [];
const dicts = j.dicts;
if (!Array.isArray(dicts))
return [];
const out = [];
for (const it of dicts) {
if (it && typeof it === 'object') {
const id = it.id;
const url = it.url;
if (typeof id === 'string' && typeof url === 'string')
out.push({ id, url });
}
}
return out;
}
// Public alias for the built-in HTTP(S) loader
export { conditionalGet as defaultLoader };