omnichron
Version:
Unified interface for web archive providers
303 lines (298 loc) • 9.26 kB
JavaScript
import { consola } from "consola";
import { loadConfig } from "c12";
import { cleanDoubleSlashes, hasProtocol, withTrailingSlash, withoutProtocol } from "ufo";
//#region node_modules/.pnpm/unstorage@1.15.0_@netlify+blobs@8.2.0_db0@0.3.2_ioredis@5.6.1/node_modules/unstorage/drivers/utils/index.mjs
function defineDriver(factory) {
return factory;
}
//#endregion
//#region node_modules/.pnpm/unstorage@1.15.0_@netlify+blobs@8.2.0_db0@0.3.2_ioredis@5.6.1/node_modules/unstorage/drivers/memory.mjs
const DRIVER_NAME = "memory";
var memory_default = defineDriver(() => {
const data = /* @__PURE__ */ new Map();
return {
name: DRIVER_NAME,
getInstance: () => data,
hasItem(key) {
return data.has(key);
},
getItem(key) {
return data.get(key) ?? null;
},
getItemRaw(key) {
return data.get(key) ?? null;
},
setItem(key, value) {
data.set(key, value);
},
setItemRaw(key, value) {
data.set(key, value);
},
removeItem(key) {
data.delete(key);
},
getKeys() {
return [...data.keys()];
},
clear() {
data.clear();
},
dispose() {
data.clear();
}
};
});
//#endregion
//#region src/config.ts
const getDefaultConfig = () => ({
storage: {
driver: memory_default(),
cache: true,
ttl: 7 * 24 * 60 * 60 * 1e3,
prefix: "omnichron"
},
performance: {
concurrency: 3,
batchSize: 20,
timeout: 1e4,
retries: 1
}
});
let cachedConfig;
/**
* Load Omnichron configuration from all available sources
*/
async function resolveConfig(options = {}) {
if (cachedConfig) return cachedConfig;
const defaults = getDefaultConfig();
const { config } = await loadConfig({
name: "omnichron",
defaults,
defaultConfig: options.defaults || void 0,
overrides: options.overrides || void 0,
envName: options.envName || "development",
cwd: options.cwd,
configFile: options.configFile,
rcFile: options.rcFile === void 0 ? ".omnichron" : options.rcFile,
packageJson: true
});
const resolvedConfig = await postProcessConfig(config, defaults);
cachedConfig = resolvedConfig;
return resolvedConfig;
}
/**
* Apply additional configuration processing and validation
*/
async function postProcessConfig(config, defaults) {
if (!config.storage) config.storage = { ...defaults.storage };
if (!config.performance) config.performance = { ...defaults.performance };
if (!config.storage.prefix) config.storage.prefix = defaults.storage.prefix;
if (!config.storage.driver) config.storage.driver = memory_default();
return config;
}
/**
* Reset the cached configuration
*/
function resetConfig() {
cachedConfig = void 0;
}
/**
* Get the current configuration or resolve it if not already loaded
*/
async function getConfig(options) {
if (cachedConfig) return cachedConfig;
return resolveConfig(options);
}
//#endregion
//#region src/utils/_utils.ts
async function processInParallel(items, processFunction, options = {}) {
const config = await getConfig();
const concurrency = options.concurrency ?? config.performance.concurrency;
const batchSize = options.batchSize ?? config.performance.batchSize;
if (items.length <= concurrency) return Promise.all(items.map((item) => processFunction(item)));
const results = [];
for (let i = 0; i < items.length; i += batchSize) {
const batch = items.slice(i, i + batchSize);
const batchResults = await processBatch(batch, concurrency);
results.push(...batchResults);
}
return results;
async function processBatch(batch, limit) {
const batchResults = [];
const pendingPromises = [];
const queue = [...batch];
while (pendingPromises.length < limit && queue.length > 0) {
const item = queue.shift();
const promise = processFunction(item).then((result) => {
batchResults.push(result);
}).catch((error) => {
consola.error("Parallel processing error:", error);
});
pendingPromises.push(promise);
}
while (queue.length > 0) {
await Promise.race(pendingPromises);
const stillPending = [];
const item = queue.shift();
const promise = processFunction(item).then((result) => {
batchResults.push(result);
}).catch((error) => {
consola.error("Parallel processing error:", error);
});
stillPending.push(promise);
pendingPromises.length = 0;
pendingPromises.push(...stillPending);
}
await Promise.all(pendingPromises);
return batchResults;
}
}
/**
* Converts a Wayback Machine timestamp to ISO8601 format
* @param timestamp Wayback timestamp (YYYYMMDDhhmmss)
* @returns ISO8601 formatted timestamp
*/
function waybackTimestampToISO(timestamp) {
return timestamp.length >= 14 ? `${timestamp.slice(0, 4)}-${timestamp.slice(4, 6)}-${timestamp.slice(6, 8)}T${timestamp.slice(8, 10)}:${timestamp.slice(10, 12)}:${timestamp.slice(12, 14)}Z` : new Date().toISOString();
}
/**
* Normalizes a domain string for search queries
* @param domain The domain or URL to normalize
* @param appendWildcard Whether to append a wildcard for prefix matching
* @returns Normalized domain string
*/
function normalizeDomain(domain, appendWildcard = true) {
const normalizedDomain = hasProtocol(domain) ? withoutProtocol(domain) : domain;
if (domain.includes("*")) return normalizedDomain;
return appendWildcard ? withTrailingSlash(normalizedDomain) + "*" : normalizedDomain;
}
/**
* Creates a standardized success response object
* @param pages Array of archived pages
* @param source Source identifier for the provider
* @param metadata Additional metadata to include
* @returns Standardized ArchiveResponse object
*/
function createSuccessResponse(pages, source, metadata = {}) {
return {
success: true,
pages,
_meta: {
source,
provider: source,
...metadata
}
};
}
/**
* Creates a standardized error response object
* @param error Error object or message
* @param source Source identifier for the provider
* @param metadata Additional metadata to include
* @returns Standardized ArchiveResponse error object
*/
function createErrorResponse(error, source, metadata = {}) {
return {
success: false,
pages: [],
error: typeof error === "string" ? error : error.message ?? String(error),
_meta: {
source,
provider: source,
errorDetails: error,
errorName: error instanceof Error ? error.name : "UnknownError",
...metadata
}
};
}
/**
* Creates common fetch options with standard defaults
* @param baseURL Base URL for the API
* @param params Query parameters
* @param options Additional options
* @returns FetchOptions object
*/
async function createFetchOptions(baseURL, params = {}, options = {}) {
const config = await getConfig();
return {
method: "GET",
baseURL,
params,
retry: options.retries ?? config.performance.retries,
timeout: options.timeout ?? config.performance.timeout,
retryDelay: 300,
retryStatusCodes: [
408,
409,
425,
429,
500,
502,
503,
504
],
onResponseError: ({ request, response, options: options$1 }) => {
consola.error(`[fetch error] ${options$1.method} ${request} failed with status ${response.status}`);
},
...options
};
}
/**
* Merges initial options with request options, preferring request options
* @param initOptions Initial options provided during provider creation
* @param reqOptions Request-specific options
* @returns Merged options object
*/
async function mergeOptions(initOptions = {}, reqOptions = {}) {
const config = await getConfig();
const defaultOptions = {
concurrency: config.performance.concurrency,
batchSize: config.performance.batchSize,
timeout: config.performance.timeout,
retries: config.performance.retries,
cache: config.storage.cache,
ttl: config.storage.ttl
};
return {
...defaultOptions,
...initOptions,
...reqOptions
};
}
/**
* Maps CDX server API response rows to ArchivedPage objects.
* @param dataRows Array of rows from CDX API, excluding header.
* @param snapshotBaseUrl Base URL for snapshot (including path segment).
* @param providerSlug Provider identifier used for metadata typing.
* @param options Performance options for processing.
* @returns Array of ArchivedPage objects.
*/
async function mapCdxRows(dataRows, snapshotBaseUrl, providerSlug = "wayback", options = {}) {
const config = await getConfig();
const batchSize = options.batchSize ?? config.performance.batchSize;
if (dataRows.length <= batchSize) return dataRows.map((row) => rowToArchivedPage(row));
const results = [];
for (let i = 0; i < dataRows.length; i += batchSize) {
const batch = dataRows.slice(i, i + batchSize);
results.push(...batch.map((row) => rowToArchivedPage(row)));
}
return results;
function rowToArchivedPage([rawUrl, rawTimestamp, rawStatus]) {
const originalUrl = cleanDoubleSlashes(rawUrl ?? "");
const timestampRaw = rawTimestamp ?? "";
const isoTimestamp = waybackTimestampToISO(timestampRaw);
const snapUrl = `${snapshotBaseUrl}/${timestampRaw}/${originalUrl}`;
return {
url: originalUrl,
timestamp: isoTimestamp,
snapshot: snapUrl,
_meta: {
timestamp: timestampRaw,
status: Number.parseInt(rawStatus ?? "0", 10),
provider: providerSlug
}
};
}
}
//#endregion
export { createErrorResponse, createFetchOptions, createSuccessResponse, getConfig, mapCdxRows, memory_default, mergeOptions, normalizeDomain, processInParallel, resetConfig, resolveConfig, waybackTimestampToISO };