@promptbook/remote-server/esm/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts

Promptbook: Create persistent AI agents that turn your company's scattered knowledge into action

import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
import type { string_file_extension } from '../../../types/string_sha256';
import type { ScraperSourceHandler } from '../Scraper';
import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
/**
 * Type representing source information for generating intermediate storage.
 * Contains essential file and URL information needed to create a deterministic storage path.
 *
 * @private internal utility of `getScraperIntermediateSource`
 */
type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
/**
 * Options for intermediate source generation and management.
 * Configures how intermediate files are named, stored, and cleaned up.
 *
 * @private internal utility of `getScraperIntermediateSource`
 */
type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'intermediateFilesStrategy' | 'isVerbose'>> & {
    readonly extension: string_file_extension;
};
/**
 * Retrieves an intermediate source for a scraper based on the knowledge source.
 * Manages the caching and retrieval of intermediate scraper results for optimized performance.
 *
 * @private as internal utility for scrapers
 */
export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
export {};