omnichron
Version:
Unified interface for web archive providers
284 lines (280 loc) • 9.53 kB
JavaScript
import { getConfig, memory_default, mergeOptions, processInParallel, resetConfig, resolveConfig } from "./_utils.mjs";
import { createStorage } from "unstorage";
import { consola } from "consola";
//#region src/storage.ts
const storage = createStorage({ driver: memory_default() });
/**
* Initialize storage with configuration values
* This is called internally when needed
*/
async function initStorage() {
const config = await getConfig();
if (config.storage.driver) Object.assign(storage, createStorage({ driver: config.storage.driver }));
}
/**
* Generate a storage key for a domain request
*/
function generateStorageKey(provider, domain, options) {
const providerKey = provider.slug ?? provider.name;
const prefix = getStoragePrefix();
const baseKey = `${prefix}:${providerKey}:${domain}`;
return options?.limit ? `${baseKey}:${options.limit}` : baseKey;
}
/**
* Get the current storage prefix
*/
function getStoragePrefix() {
return storage.options?.prefix || "omnichron";
}
/**
* Get stored response if available
*/
async function getStoredResponse(provider, domain, options) {
if (options?.cache === false) return void 0;
if (!storage.options) await initStorage();
const key = generateStorageKey(provider, domain, options);
try {
const cachedData = await storage.getItem(key);
if (cachedData) try {
const parsedData = typeof cachedData === "string" ? JSON.parse(cachedData) : cachedData;
return {
...parsedData,
fromCache: true
};
} catch (parseError) {
consola.error(`Storage parse error for ${key}:`, parseError);
}
} catch (error) {
consola.error(`Storage read error for ${key}:`, error);
}
return void 0;
}
/**
* Store response in storage
*/
async function storeResponse(provider, domain, response, options) {
if (options?.cache === false || !response.success) return;
if (!storage.options) await initStorage();
const key = generateStorageKey(provider, domain, options);
try {
const { fromCache: _fromCache,...storableResponse } = response;
await storage.setItem(key, JSON.stringify(storableResponse));
} catch (error) {
consola.error(`Storage write error for ${key}:`, error);
}
}
/**
* Clear stored responses for a specific provider
*/
async function clearProviderStorage(provider) {
try {
if (!storage.options) await initStorage();
const providerKey = typeof provider === "string" ? provider : provider.slug ?? provider.name;
const _prefix = `${getStoragePrefix()}:${providerKey}`;
await storage.clear();
} catch (error) {
const providerName = typeof provider === "string" ? provider : provider.name;
consola.error(`Failed to clear storage for provider ${providerName}:`, error);
}
}
/**
* Configure storage options and driver
* @deprecated Use config file or options passed to createArchive instead
*/
async function configureStorage(options = {}) {
const config = await getConfig();
if (options.driver) config.storage.driver = options.driver;
if (options.ttl !== void 0) config.storage.ttl = options.ttl;
if (options.cache !== void 0) config.storage.cache = options.cache;
if (options.prefix !== void 0) {
storage.options = storage.options || {};
storage.options.prefix = options.prefix;
}
if (options.driver) {
const newStorage = createStorage({ driver: options.driver });
newStorage.options = newStorage.options || {};
newStorage.options.prefix = storage.options?.prefix || config.storage.prefix;
Object.assign(storage, newStorage);
}
}
//#endregion
//#region src/archive.ts
/**
* Create a unified archive client that wraps one or multiple providers.
* Supports lazy loading and asynchronous provider initialization.
*
* @param providers - Single provider, array of providers, or Promise(s) resolving to provider(s)
* @param options - Default options applied to all queries (limit, cache, ttl, concurrency, etc.)
* @returns Archive client with methods for fetching and managing archive data
*
* @example
* ```js
* // Single provider
* const waybackArchive = createArchive(providers.wayback())
*
* // Multiple providers
* const multiArchive = createArchive([
* providers.wayback(),
* providers.archiveToday()
* ])
*
* // With options
* const archive = createArchive(providers.all(), {
* limit: 10,
* cache: true,
* ttl: 3600000, // 1 hour cache TTL
* concurrency: 3
* })
* ```
*/
function createArchive(providers$1, options) {
let resolvedProviders = void 0;
/**
* Resolves and caches the provider promises.
* Ensures providers are only resolved once and then cached for future use.
*
* @returns Promise resolving to array of all initialized providers
* @internal
*/
async function getProviders() {
if (resolvedProviders) return resolvedProviders;
const result = await Promise.resolve(providers$1);
resolvedProviders = Array.isArray(result) ? result : [result];
return resolvedProviders;
}
/**
* Fetches data from a single provider with built-in caching.
* Attempts to read from cache first, then falls back to fresh data.
*
* @param provider - The archive provider to query
* @param domain - The domain to search for archives
* @param requestOptions - Options for this specific request
* @returns Promise resolving to provider's response or error response
* @internal
*/
async function fetchFromProvider(provider, domain, requestOptions) {
if (requestOptions.cache !== false) {
const cached = await getStoredResponse(provider, domain, requestOptions);
if (cached) return cached;
}
try {
const response = await provider.getSnapshots(domain, requestOptions);
if (response.success && requestOptions.cache !== false) await storeResponse(provider, domain, response, requestOptions);
return response;
} catch (error) {
return {
success: false,
pages: [],
error: error instanceof Error ? error.message : String(error),
_meta: {
source: provider.name,
provider: provider.name,
errorDetails: error
}
};
}
}
/**
* Combines results from multiple providers into a single response.
* Merges pages, handles errors, applies sorting and pagination.
*
* @param responses - Array of responses from different providers
* @param limit - Optional limit on number of pages to return
* @returns Combined archive response with merged pages and metadata
* @internal
*/
function combineResults(responses, limit) {
const allPages = [];
const errors = [];
let anySuccess = false;
for (const response of responses) if (response.success) {
anySuccess = true;
allPages.push(...response.pages);
} else if (response.error) errors.push(response.error);
allPages.sort((a, b) => {
return new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
});
const limitedPages = limit ? allPages.slice(0, limit) : allPages;
const providersList = responses.map((r) => r._meta?.provider || "unknown").filter(Boolean);
return {
success: anySuccess,
pages: limitedPages,
error: anySuccess ? void 0 : errors.join("; "),
_meta: {
source: "multiple",
provider: providersList.join(","),
providerCount: providersList.length,
errors: errors.length > 0 ? errors : void 0
}
};
}
const archive = {
options,
async getSnapshots(domain, listOptions) {
const mergedOptions = await mergeOptions(options, listOptions);
const providerArray = await getProviders();
if (providerArray.length === 1) return fetchFromProvider(providerArray[0], domain, mergedOptions);
const responses = await processInParallel(providerArray, (provider) => fetchFromProvider(provider, domain, mergedOptions), {
concurrency: mergedOptions.concurrency,
batchSize: mergedOptions.batchSize
});
return combineResults(responses, mergedOptions.limit);
},
async getPages(domain, listOptions) {
const res = await this.getSnapshots(domain, listOptions);
if (!res.success) throw new Error(res.error ?? "Failed to fetch archive snapshots");
return res.pages;
},
async use(provider) {
const resolvedProvider = await Promise.resolve(provider);
const currentProviders = await getProviders();
resolvedProviders = [...currentProviders, resolvedProvider];
return this;
},
async useAll(newProviders) {
const resolvedNewProviders = await Promise.all(newProviders.map((p) => Promise.resolve(p)));
const currentProviders = await getProviders();
resolvedProviders = [...currentProviders, ...resolvedNewProviders];
return this;
}
};
return archive;
}
//#endregion
//#region src/providers/index.ts
/**
* Provider factory with lazy-loading for optimized tree-shaking.
* Only loads the providers that are actually used.
*/
const providers = {
async wayback(options) {
const { default: create } = await import("./wayback.mjs");
return create(options);
},
async archiveToday(options) {
const { default: create } = await import("./archive-today.mjs");
return create(options);
},
async permacc(options) {
const { default: create } = await import("./permacc.mjs");
return create(options);
},
async commoncrawl(options) {
const { default: create } = await import("./commoncrawl.mjs");
return create(options);
},
async webcite(options) {
const { default: create } = await import("./webcite.mjs");
return create(options);
},
async all(options) {
return Promise.all([
this.wayback(options),
this.archiveToday(options),
this.commoncrawl(options),
this.webcite(options)
]);
}
};
//#endregion
export { clearProviderStorage, configureStorage, createArchive, getConfig, providers, resetConfig, resolveConfig, storage };