@promptbook/remote-client
Version:
Promptbook: Turn your company's scattered knowledge into AI ready books
36 lines (35 loc) • 1.84 kB
TypeScript
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
import type { string_file_extension } from '../../../types/typeAliases';
import type { ScraperSourceHandler } from '../Scraper';
import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
/**
* Type representing source information for generating intermediate storage.
* Contains essential file and URL information needed to create a deterministic storage path.
*
* @private internal utility of `getScraperIntermediateSource`
*/
type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
/**
* Options for intermediate source generation and management.
* Configures how intermediate files are named, stored, and cleaned up.
*
* @private internal utility of `getScraperIntermediateSource`
*/
type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'intermediateFilesStrategy' | 'isVerbose'>> & {
readonly extension: string_file_extension;
};
/**
* Retrieves an intermediate source for a scraper based on the knowledge source.
* Manages the caching and retrieval of intermediate scraper results for optimized performance.
*
* @private as internal utility for scrapers
*/
export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
export {};
/**
* Note: Not using `FileCacheStorage` for two reasons:
* 1) Need to store more than serialized JSONs
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
* TODO: [🐱🐉][🧠] Make some smart crop
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
*/