omnichron
Version:
Unified interface for web archive providers
314 lines (307 loc) • 8.91 kB
text/typescript
import { Driver } from "unstorage";
//#region src/types.d.ts
interface ArchiveOptions {
limit?: number;
cache?: boolean;
ttl?: number;
concurrency?: number;
batchSize?: number;
timeout?: number;
retries?: number;
apiKey?: string;
}
interface ArchiveMetadata {
[key: string]: unknown;
timestamp?: string;
status?: number;
}
interface WaybackMetadata extends ArchiveMetadata {
timestamp: string;
status: number;
provider: string;
}
interface CommonCrawlMetadata extends ArchiveMetadata {
timestamp: string;
status: number;
digest?: string;
mime?: string;
length?: string;
collection: string;
provider: string;
}
interface PermaccMetadata extends Omit<ArchiveMetadata, "status"> {
guid: string;
title?: string;
status?: string;
created_by?: string;
}
interface ArchiveTodayMetadata extends ArchiveMetadata {
hash: string;
raw_date?: string;
position?: number;
}
interface WebCiteMetadata extends ArchiveMetadata {
requestId: string;
position?: number;
}
interface UkWebArchiveMetadata extends ArchiveMetadata {
timestamp: string;
status: number;
}
interface MementoTimeMetadata extends ArchiveMetadata {
originalTimestamp: string;
source: string;
position?: number;
provider: string;
}
interface ArchivedPage {
url: string;
timestamp: string;
snapshot: string;
_meta: ArchivedPageMetadata;
}
interface ArchivedPageMetadata {
timestamp?: string;
status?: number | string;
provider?: string;
source?: string;
[key: string]: unknown;
}
interface ResponseMetadata {
source: string;
provider: string;
errorDetails?: unknown;
errorName?: string;
queryParams?: Record<string, string>;
[key: string]: unknown;
}
interface ArchiveResponse {
success: boolean;
pages: ArchivedPage[];
error?: string;
_meta?: ResponseMetadata;
fromCache?: boolean;
}
type ArchiveResult = {
success: true
pages: ArchivedPage[]
_meta?: ResponseMetadata
fromCache?: boolean
} | {
success: false
error: string
pages: never[]
_meta?: ResponseMetadata
fromCache?: boolean
};
interface ArchiveProvider {
name: string;
slug?: string;
getSnapshots: (domain: string, options?: ArchiveOptions) => Promise<ArchiveResponse>;
}
type ReadonlyArchivedPage = Readonly<ArchivedPage>;
type ReadonlyArchiveResponse = Readonly<ArchiveResponse>;
/**
* Interface for Archive instances
* Defines the public API that all archive implementations must provide
*/
interface ArchiveInterface {
readonly options?: ArchiveOptions;
getSnapshots(domain: string, options?: ArchiveOptions): Promise<ArchiveResponse>;
getPages(domain: string, options?: ArchiveOptions): Promise<ArchivedPage[]>;
use(provider: ArchiveProvider | Promise<ArchiveProvider>): Promise<ArchiveInterface>;
useAll(providers: (ArchiveProvider | Promise<ArchiveProvider>)[]): Promise<ArchiveInterface>;
onBeforeRequest?(domain: string, options: ArchiveOptions): Promise<void>;
onAfterResponse?(response: ArchiveResponse): Promise<void>;
}
//#endregion
//#region src/archive.d.ts
/**
* Create a unified archive client that wraps one or multiple providers.
* Supports lazy loading and asynchronous provider initialization.
*
* @param providers - Single provider, array of providers, or Promise(s) resolving to provider(s)
* @param options - Default options applied to all queries (limit, cache, ttl, concurrency, etc.)
* @returns Archive client with methods for fetching and managing archive data
*
* @example
* ```js
* // Single provider
* const waybackArchive = createArchive(providers.wayback())
*
* // Multiple providers
* const multiArchive = createArchive([
* providers.wayback(),
* providers.archiveToday()
* ])
*
* // With options
* const archive = createArchive(providers.all(), {
* limit: 10,
* cache: true,
* ttl: 3600000, // 1 hour cache TTL
* concurrency: 3
* })
* ```
*/
declare function createArchive(providers: ArchiveProvider | ArchiveProvider[] | Promise<ArchiveProvider> | Promise<ArchiveProvider[]>, options?: ArchiveOptions): {
options?: ArchiveOptions
getSnapshots(domain: string, listOptions?: ArchiveOptions): Promise<ArchiveResponse>
getPages(domain: string, listOptions?: ArchiveOptions): Promise<ArchivedPage[]>
use(provider: ArchiveProvider | Promise<ArchiveProvider>): Promise<any>
useAll(newProviders: (ArchiveProvider | Promise<ArchiveProvider>)[]): Promise<any>
};
//#endregion
//#region src/_providers.d.ts
interface WaybackOptions extends ArchiveOptions {
collapse?: string;
filter?: string;
}
interface ArchiveTodayOptions extends ArchiveOptions {
maxRedirects?: number;
}
interface PermaccOptions extends ArchiveOptions {
apiKey: string;
}
interface CommonCrawlOptions extends ArchiveOptions {
collection?: string;
}
type WebCiteOptions = ArchiveOptions;
//#endregion
//#region src/providers/index.d.ts
/**
* Provider factory with lazy-loading for optimized tree-shaking.
* Only loads the providers that are actually used.
*/
declare const providers: {
/**
* Creates a Wayback Machine provider.
* @param options - Configuration options for the Wayback Machine provider
* @returns The Wayback Machine provider
* @example
* ```js
* const waybackProvider = providers.wayback({ limit: 100 })
* ```
*/
wayback(options?: WaybackOptions): Promise<ArchiveProvider>
/**
* Creates an Archive.today provider.
* @param options - Configuration options for the Archive.today provider
* @returns The Archive.today provider
* @example
* ```js
* const archiveTodayProvider = providers.archiveToday({ maxRedirects: 5 })
* ```
*/
archiveToday(options?: ArchiveTodayOptions): Promise<ArchiveProvider>
/**
* Creates a Perma.cc provider.
* @param options - Configuration options for the Perma.cc provider (requires apiKey)
* @returns The Perma.cc provider
* @example
* ```js
* const permaccProvider = providers.permacc({ apiKey: 'your-api-key' })
* ```
*/
permacc(options?: PermaccOptions): Promise<ArchiveProvider>
/**
* Creates a Common Crawl provider.
* @param options - Configuration options for the Common Crawl provider
* @returns The Common Crawl provider
* @example
* ```js
* const commoncrawlProvider = providers.commoncrawl({ collection: 'CC-MAIN-2023-50' })
* ```
*/
commoncrawl(options?: CommonCrawlOptions): Promise<ArchiveProvider>
/**
* Creates a WebCite provider.
* @param options - Configuration options for the WebCite provider
* @returns The WebCite provider
* @example
* ```js
* const webciteProvider = providers.webcite({ timeout: 10000 })
* ```
*/
webcite(options?: WebCiteOptions): Promise<ArchiveProvider>
/**
* Helper to initialize all commonly used providers at once.
* Note: Perma.cc is excluded as it requires an API key.
* @param options - Common configuration options for all providers
* @returns An array of all common providers
* @example
* ```js
* const allProviders = providers.all({ timeout: 15000 })
* const archive = createArchive(allProviders)
* ```
*/
all(options?: ArchiveOptions): Promise<ArchiveProvider[]>
};
//#endregion
//#region src/storage.d.ts
declare const storage: Storage & {
options?: {
prefix?: string
}
};
/**
* Clear stored responses for a specific provider
*/
declare function clearProviderStorage(provider: string | {
name: string
slug?: string
}): Promise<void>;
/**
* Configure storage options and driver
* @deprecated Use config file or options passed to createArchive instead
*/
declare function configureStorage(options?: {
driver?: any
ttl?: number
cache?: boolean
prefix?: string
}): Promise<void>;
//#endregion
//#region src/config.d.ts
/**
* Configuration options for Omnichron
*/
interface OmnichronConfig {
storage: {
driver?: Driver
cache?: boolean
ttl?: number
prefix?: string
};
performance: {
concurrency?: number
batchSize?: number
timeout?: number
retries?: number
};
$env?: Record<string, OmnichronConfig>;
$development?: OmnichronConfig;
$production?: OmnichronConfig;
$test?: OmnichronConfig;
}
/**
* Load Omnichron configuration from all available sources
*/
declare function resolveConfig(options?: {
cwd?: string
defaults?: Partial<OmnichronConfig>
overrides?: Partial<OmnichronConfig>
envName?: string | false
configFile?: string
rcFile?: string
}): Promise<OmnichronConfig>;
/**
* Reset the cached configuration
*/
declare function resetConfig(): void;
/**
* Get the current configuration or resolve it if not already loaded
*/
declare function getConfig(options?: Parameters<typeof resolveConfig>[0]): Promise<OmnichronConfig>;
//#endregion
export { ArchiveInterface, ArchiveMetadata, ArchiveOptions, ArchiveProvider, ArchiveResponse, ArchiveResult, ArchiveTodayMetadata, ArchivedPage, ArchivedPageMetadata, CommonCrawlMetadata, MementoTimeMetadata, PermaccMetadata, ReadonlyArchiveResponse, ReadonlyArchivedPage, ResponseMetadata, UkWebArchiveMetadata, WaybackMetadata, WebCiteMetadata, clearProviderStorage, configureStorage, createArchive, getConfig, providers, resetConfig, resolveConfig, storage };