crewai-ts
Version:
TypeScript port of crewAI for agent-based workflows
97 lines • 2.91 kB
TypeScript
/**
* ScrapeWebsiteTool implementation
* Provides web scraping capabilities for agents
* Optimized for performance, handling different content types, and memory efficiency
*/
import { z } from 'zod';
declare const scrapeWebsiteSchema: z.ZodObject<{
url: z.ZodString;
selector: z.ZodOptional<z.ZodString>;
includeLinks: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
includeTables: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
includeImages: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
timeout: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
userAgent: z.ZodOptional<z.ZodString>;
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
cookies: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
waitForSelector: z.ZodOptional<z.ZodString>;
waitTime: z.ZodOptional<z.ZodNumber>;
}, "strip", z.ZodTypeAny, {
timeout?: number;
url?: string;
selector?: string;
includeLinks?: boolean;
includeTables?: boolean;
includeImages?: boolean;
userAgent?: string;
headers?: Record<string, string>;
cookies?: Record<string, string>;
waitForSelector?: string;
waitTime?: number;
}, {
timeout?: number;
url?: string;
selector?: string;
includeLinks?: boolean;
includeTables?: boolean;
includeImages?: boolean;
userAgent?: string;
headers?: Record<string, string>;
cookies?: Record<string, string>;
waitForSelector?: string;
waitTime?: number;
}>;
type ScrapeWebsiteInput = z.infer<typeof scrapeWebsiteSchema>;
/**
* Result of a web scraping operation
*/
export interface ScrapeWebsiteResult {
content: string;
title?: string;
links?: Array<{
text: string;
url: string;
}>;
tables?: Array<{
headers: string[];
rows: string[][];
}>;
images?: Array<{
alt: string;
src: string;
}>;
url: string;
statusCode?: number;
contentType?: string;
error?: string;
}
/**
* Options for configuring the ScrapeWebsiteTool
*/
export interface ScrapeWebsiteToolOptions {
customScraper?: (input: ScrapeWebsiteInput) => Promise<ScrapeWebsiteResult>;
cacheResults?: boolean;
cacheExpiration?: number;
timeoutMs?: number;
maxRetries?: number;
userAgent?: string;
headers?: Record<string, string>;
}
/**
* Creates a web scraping tool with the specified configuration
*/
export declare function createScrapeWebsiteTool(options?: ScrapeWebsiteToolOptions): import("../StructuredTool.js").StructuredTool<{
timeout?: number;
url?: string;
selector?: string;
includeLinks?: boolean;
includeTables?: boolean;
includeImages?: boolean;
userAgent?: string;
headers?: Record<string, string>;
cookies?: Record<string, string>;
waitForSelector?: string;
waitTime?: number;
}, ScrapeWebsiteResult>;
export {};
//# sourceMappingURL=ScrapeWebsiteTool.d.ts.map