UNPKG

@clduab11/gemini-flow

Version:

Revolutionary AI agent swarm coordination platform with Google Services integration, multimedia processing, and production-ready monitoring. Features 8 Google AI services, quantum computing capabilities, and enterprise-grade security.

1,010 lines (889 loc) 27.9 kB
/** * Intelligent Form Filler * * AI-powered form filling with smart field detection, validation, * and adaptive interaction patterns */ import { EventEmitter } from "events"; import { Logger } from "../../utils/logger.js"; import { FormFillingConfig, ValidationRule, ElementDetectionConfig, CaptchaSolvingConfig, AntiDetectionConfig, FormStructure, FormField, FormValidation, SubmissionInfo, BrowserTab, IntegrationBaseError, } from "./types.js"; import { BaseIntegration, HealthStatus } from "../shared/types.js"; export interface FormFillingRequest { id: string; tab: BrowserTab; formData: Record<string, any>; options: FormFillingOptions; validation: FormFillingValidation; } export interface FormFillingOptions { skipOptionalFields: boolean; useSmartDefaults: boolean; enableValidation: boolean; submitForm: boolean; captchaSolving: boolean; humanLikeDelay: boolean; fieldPriority: Record<string, number>; } export interface FormFillingValidation { validateBeforeSubmit: boolean; requiredFields: string[]; customValidators: Map<string, (value: any) => boolean>; skipInvalidFields: boolean; } export interface FormFillingResult { success: boolean; formId: string; fieldsProcessed: number; fieldsFilled: number; validationErrors: string[]; submissionResult?: SubmissionResult; duration: number; metadata: Record<string, any>; } export interface SubmissionResult { submitted: boolean; response: any; redirectUrl?: string; errors: string[]; } export interface FieldMapping { sourceField: string; targetSelector: string; transformer?: (value: any) => any; validator?: (value: any) => boolean; priority: number; } export interface SmartDefault { pattern: RegExp; generator: () => any; category: string; } export class IntelligentFormFiller extends BaseIntegration { private config: FormFillingConfig; private fieldMappings: Map<string, FieldMapping> = new Map(); private smartDefaults: SmartDefault[] = []; private formCache: Map<string, FormStructure> = new Map(); // Performance metrics private fillerMetrics = { formsProcessed: 0, fieldsProcessed: 0, successRate: 0, avgProcessingTime: 0, validationErrors: 0, captchasSolved: 0, }; constructor(config: FormFillingConfig) { super({ id: "intelligent-form-filler", name: "Intelligent Form Filler", version: "1.0.0", enabled: config.enabled, dependencies: ["puppeteer"], features: { aiAssisted: config.aiAssisted, dataValidation: config.dataValidation, smartDefaults: config.smartDefaults, }, performance: { maxConcurrentOperations: 5, timeoutMs: 30000, retryAttempts: 3, cacheEnabled: true, cacheTTLMs: 3600000, metricsEnabled: true, }, security: { encryption: true, validateOrigins: true, allowedHosts: [], tokenExpiration: 3600, auditLogging: true, }, storage: { provider: "memory", encryption: true, compression: false, }, }); this.config = config; this.logger = new Logger("IntelligentFormFiller"); } async initialize(): Promise<void> { try { this.status = "initializing"; this.logger.info("Initializing Intelligent Form Filler"); // Initialize field mappings this.initializeFieldMappings(); // Initialize smart defaults this.initializeSmartDefaults(); // Load validation rules this.loadValidationRules(); this.status = "ready"; this.logger.info("Intelligent Form Filler initialized successfully"); this.emit("initialized", { timestamp: new Date() }); } catch (error) { this.status = "error"; const fillerError = new IntegrationBaseError( `Failed to initialize Intelligent Form Filler: ${error.message}`, "INIT_FAILED", "IntelligentFormFiller", "critical", false, { originalError: error.message }, ); this.emitError(fillerError); throw fillerError; } } async shutdown(): Promise<void> { try { this.logger.info("Shutting down Intelligent Form Filler"); this.status = "shutdown"; // Clear caches this.formCache.clear(); this.fieldMappings.clear(); this.logger.info("Intelligent Form Filler shutdown complete"); this.emit("shutdown", { timestamp: new Date() }); } catch (error) { this.logger.error("Error during Intelligent Form Filler shutdown", error); throw error; } } async healthCheck(): Promise<HealthStatus> { try { // Check if we can perform basic operations if (this.fieldMappings.size === 0 || this.smartDefaults.length === 0) { return "warning"; } return "healthy"; } catch (error) { this.logger.error("Health check failed", error); return "critical"; } } getMetrics(): Record<string, number> { return { ...this.fillerMetrics, fieldMappings: this.fieldMappings.size, smartDefaults: this.smartDefaults.length, cachedForms: this.formCache.size, }; } // === MAIN FORM FILLING METHOD === async fillForm(request: FormFillingRequest): Promise<FormFillingResult> { const startTime = performance.now(); try { this.logger.info("Starting intelligent form filling", { requestId: request.id, tabId: request.tab.id, fieldsToFill: Object.keys(request.formData).length, }); // Analyze form structure const formStructure = await this.analyzeFormStructure(request.tab); // Map form data to fields const fieldMappings = this.mapFormData(request.formData, formStructure); // Fill form fields intelligently const fillResult = await this.performIntelligentFilling( request.tab, fieldMappings, request.options, ); // Validate form if required let validationErrors: string[] = []; if (request.validation.validateBeforeSubmit) { validationErrors = await this.validateForm( request.tab, formStructure, request.validation, ); } // Submit form if requested and validation passed let submissionResult: SubmissionResult | undefined; if (request.options.submitForm && validationErrors.length === 0) { submissionResult = await this.submitForm(request.tab, formStructure); } const duration = performance.now() - startTime; this.fillerMetrics.formsProcessed++; this.fillerMetrics.avgProcessingTime = (this.fillerMetrics.avgProcessingTime + duration) / 2; const result: FormFillingResult = { success: validationErrors.length === 0, formId: formStructure.selector, fieldsProcessed: fillResult.fieldsProcessed, fieldsFilled: fillResult.fieldsFilled, validationErrors, submissionResult, duration, metadata: { requestId: request.id, aiAssisted: this.config.aiAssisted, smartDefaults: request.options.useSmartDefaults, formComplexity: formStructure.fields.length, }, }; this.logger.info("Form filling completed", { requestId: request.id, success: result.success, fieldsProcessed: result.fieldsProcessed, fieldsFilled: result.fieldsFilled, duration: result.duration, }); this.emit("form_filled", { request, result, timestamp: new Date() }); return result; } catch (error) { const duration = performance.now() - startTime; const fillingError = new IntegrationBaseError( `Form filling failed: ${error.message}`, "FORM_FILLING_FAILED", "IntelligentFormFiller", "medium", true, { requestId: request.id, tabId: request.tab.id }, ); this.emitError(fillingError); return { success: false, formId: "unknown", fieldsProcessed: 0, fieldsFilled: 0, validationErrors: [error.message], duration, metadata: { requestId: request.id, failed: true }, }; } } // === FORM ANALYSIS === async analyzeFormStructure(tab: BrowserTab): Promise<FormStructure> { try { const page = tab.page; const formAnalysis = await page.evaluate(() => { const forms = Array.from(document.querySelectorAll("form")); if (forms.length === 0) { throw new Error("No forms found on page"); } // Analyze the most prominent form const form = forms[0]; // TODO: Improve form selection logic const fields: any[] = []; const inputs = form.querySelectorAll("input, select, textarea"); inputs.forEach((input: any) => { const field = { name: input.name || input.id || "", type: input.type || input.tagName.toLowerCase(), selector: this.generateSelector(input), required: input.required || input.hasAttribute("required"), validation: this.extractValidationRules(input), options: this.extractOptions(input), placeholder: input.placeholder || "", }; if (field.name || field.selector) { fields.push(field); } }); return { selector: this.generateSelector(form), method: form.method || "POST", action: form.action || "", fields, validation: { clientSide: !!form.querySelector("[required]"), serverSide: false, // Can't detect from client realTime: false, patterns: {}, }, submission: { selector: this.generateSelector( form.querySelector( '[type="submit"], button[type="submit"], input[type="submit"]', ) || form.querySelector("button"), ), method: "click", confirmationSelector: "", redirectPattern: "", }, }; }); // Cache the analyzed form this.formCache.set(tab.url, formAnalysis); return formAnalysis; } catch (error) { throw new IntegrationBaseError( `Form analysis failed: ${error.message}`, "FORM_ANALYSIS_FAILED", "IntelligentFormFiller", "medium", true, { tabId: tab.id, url: tab.url }, ); } } // === DATA MAPPING === private mapFormData( formData: Record<string, any>, formStructure: FormStructure, ): Map<string, any> { const mappings = new Map<string, any>(); for (const field of formStructure.fields) { let value = null; let mapped = false; // Try exact name match first if (formData[field.name]) { value = formData[field.name]; mapped = true; } // Try configured field mappings if (!mapped) { for (const [sourceField, mapping] of this.fieldMappings) { if (formData[sourceField] && this.isFieldMatch(field, mapping)) { value = mapping.transformer ? mapping.transformer(formData[sourceField]) : formData[sourceField]; mapped = true; break; } } } // Try smart matching by field patterns if (!mapped && this.config.aiAssisted) { value = this.smartMatchField(field, formData); mapped = !!value; } // Use smart defaults if configured if (!mapped && this.config.smartDefaults) { value = this.generateSmartDefault(field); mapped = !!value; } if (mapped && value !== null) { mappings.set(field.selector, value); } } return mappings; } private isFieldMatch(field: FormField, mapping: FieldMapping): boolean { // Check if field matches the mapping criteria return ( field.selector === mapping.targetSelector || field.name === mapping.targetSelector || field.selector.includes(mapping.targetSelector) ); } private smartMatchField( field: FormField, formData: Record<string, any>, ): any { // AI-assisted field matching based on field characteristics const fieldLower = (field.name + " " + field.placeholder).toLowerCase(); // Email fields if (fieldLower.includes("email") || field.type === "email") { return this.findValueByPattern(formData, /email|mail/i); } // Name fields if ( fieldLower.includes("name") || fieldLower.includes("first") || fieldLower.includes("last") ) { if (fieldLower.includes("first")) { return this.findValueByPattern( formData, /first.*name|given.*name|fname/i, ); } if (fieldLower.includes("last")) { return this.findValueByPattern( formData, /last.*name|family.*name|surname|lname/i, ); } return this.findValueByPattern( formData, /^name$|full.*name|display.*name/i, ); } // Phone fields if (fieldLower.includes("phone") || field.type === "tel") { return this.findValueByPattern(formData, /phone|tel|mobile|cell/i); } // Address fields if (fieldLower.includes("address")) { return this.findValueByPattern(formData, /address|street|addr/i); } // Date fields if (field.type === "date" || fieldLower.includes("date")) { return this.findValueByPattern(formData, /date|birth|dob/i); } return null; } private findValueByPattern(data: Record<string, any>, pattern: RegExp): any { for (const [key, value] of Object.entries(data)) { if (pattern.test(key)) { return value; } } return null; } private generateSmartDefault(field: FormField): any { for (const smartDefault of this.smartDefaults) { const fieldText = ( field.name + " " + field.placeholder + " " + field.type ).toLowerCase(); if (smartDefault.pattern.test(fieldText)) { return smartDefault.generator(); } } return null; } // === INTELLIGENT FILLING === private async performIntelligentFilling( tab: BrowserTab, fieldMappings: Map<string, any>, options: FormFillingOptions, ): Promise<{ fieldsProcessed: number; fieldsFilled: number }> { const page = tab.page; let fieldsProcessed = 0; let fieldsFilled = 0; // Sort fields by priority if specified const sortedFields = Array.from(fieldMappings.entries()).sort((a, b) => { const priorityA = options.fieldPriority[a[0]] || 0; const priorityB = options.fieldPriority[b[0]] || 0; return priorityB - priorityA; }); for (const [selector, value] of sortedFields) { fieldsProcessed++; try { // Wait for field to be available await page.waitForSelector(selector, { timeout: 5000 }); // Get field information const fieldInfo = await page.evaluate((sel) => { const element = document.querySelector(sel); if (!element) return null; return { tagName: element.tagName.toLowerCase(), type: (element as any).type || "", readonly: (element as any).readOnly, disabled: (element as any).disabled, value: (element as any).value || "", }; }, selector); if (!fieldInfo || fieldInfo.readonly || fieldInfo.disabled) { continue; } // Apply human-like delays if (options.humanLikeDelay) { await this.humanLikeDelay(); } // Fill field based on type await this.fillFieldByType(page, selector, value, fieldInfo); fieldsFilled++; this.fillerMetrics.fieldsProcessed++; this.logger.debug("Field filled successfully", { selector, type: fieldInfo.type, value: typeof value === "string" ? value.substring(0, 20) + "..." : value, }); } catch (error) { this.logger.warn(`Failed to fill field ${selector}`, error); // Try alternative selectors or methods const filled = await this.tryAlternativeFilling(page, selector, value); if (filled) { fieldsFilled++; } } } return { fieldsProcessed, fieldsFilled }; } private async fillFieldByType( page: any, selector: string, value: any, fieldInfo: any, ): Promise<void> { switch (fieldInfo.tagName) { case "input": await this.fillInputField(page, selector, value, fieldInfo.type); break; case "select": await this.fillSelectField(page, selector, value); break; case "textarea": await this.fillTextareaField(page, selector, value); break; default: await this.fillGenericField(page, selector, value); } } private async fillInputField( page: any, selector: string, value: any, type: string, ): Promise<void> { switch (type) { case "checkbox": case "radio": if (value) { await page.click(selector); } break; case "file": if (typeof value === "string") { const input = await page.$(selector); await input.uploadFile(value); } break; default: // Clear existing value and type new one await page.click(selector, { clickCount: 3 }); // Select all await page.type(selector, String(value), { delay: 50 }); } } private async fillSelectField( page: any, selector: string, value: any, ): Promise<void> { // Try to select by value, then by text try { await page.select(selector, String(value)); } catch (error) { // Try selecting by visible text await page.evaluate( (sel, val) => { const select = document.querySelector(sel) as HTMLSelectElement; if (select) { for (let i = 0; i < select.options.length; i++) { if ( select.options[i].text.toLowerCase().includes(val.toLowerCase()) ) { select.selectedIndex = i; select.dispatchEvent(new Event("change")); break; } } } }, selector, String(value), ); } } private async fillTextareaField( page: any, selector: string, value: any, ): Promise<void> { await page.click(selector); await page.keyboard.down("Control"); await page.keyboard.press("a"); await page.keyboard.up("Control"); await page.type(selector, String(value), { delay: 20 }); } private async fillGenericField( page: any, selector: string, value: any, ): Promise<void> { // Generic filling for custom elements await page.evaluate( (sel, val) => { const element = document.querySelector(sel) as any; if (element) { element.value = val; element.dispatchEvent(new Event("input", { bubbles: true })); element.dispatchEvent(new Event("change", { bubbles: true })); } }, selector, value, ); } private async tryAlternativeFilling( page: any, selector: string, value: any, ): Promise<boolean> { // Try alternative selectors and methods try { // Try by name attribute const nameSelector = `[name="${selector}"]`; await page.waitForSelector(nameSelector, { timeout: 1000 }); await page.type(nameSelector, String(value)); return true; } catch (error) { // Try by id try { const idSelector = `#${selector}`; await page.waitForSelector(idSelector, { timeout: 1000 }); await page.type(idSelector, String(value)); return true; } catch (error) { return false; } } } // === VALIDATION === private async validateForm( tab: BrowserTab, formStructure: FormStructure, validation: FormFillingValidation, ): Promise<string[]> { const errors: string[] = []; const page = tab.page; try { // Check required fields for (const fieldName of validation.requiredFields) { const field = formStructure.fields.find((f) => f.name === fieldName); if (field) { const value = await page.evaluate((selector) => { const element = document.querySelector(selector) as any; return element ? element.value : ""; }, field.selector); if (!value || value.trim() === "") { errors.push(`Required field '${fieldName}' is empty`); } } } // Run custom validators for (const [fieldName, validator] of validation.customValidators) { const field = formStructure.fields.find((f) => f.name === fieldName); if (field) { const value = await page.evaluate((selector) => { const element = document.querySelector(selector) as any; return element ? element.value : ""; }, field.selector); if (!validator(value)) { errors.push(`Validation failed for field '${fieldName}'`); } } } // Check built-in validation rules for (const rule of this.config.validationRules) { const field = formStructure.fields.find((f) => f.name === rule.field); if (field) { const value = await page.evaluate((selector) => { const element = document.querySelector(selector) as any; return element ? element.value : ""; }, field.selector); if (!this.validateFieldValue(value, rule)) { errors.push( `Field '${rule.field}' failed validation: ${rule.type}`, ); } } } this.fillerMetrics.validationErrors += errors.length; } catch (error) { errors.push(`Validation error: ${error.message}`); } return errors; } private validateFieldValue(value: any, rule: ValidationRule): boolean { if (rule.required && (!value || value.trim() === "")) { return false; } if (!value && !rule.required) { return true; // Optional empty field is valid } switch (rule.type) { case "email": return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value); case "phone": return /^[\d\s\-\+\(\)]+$/.test(value); case "date": return !isNaN(Date.parse(value)); case "number": return !isNaN(Number(value)); case "regex": return rule.pattern ? new RegExp(rule.pattern).test(value) : true; case "text": return typeof value === "string" && value.length > 0; default: return rule.custom ? rule.custom(value) : true; } } // === FORM SUBMISSION === private async submitForm( tab: BrowserTab, formStructure: FormStructure, ): Promise<SubmissionResult> { const page = tab.page; try { this.logger.info("Submitting form", { formSelector: formStructure.selector, }); // Handle CAPTCHA if present if (this.config.enabled) { await this.handleCaptcha(page); } // Click submit button const submitSelector = formStructure.submission.selector; await page.waitForSelector(submitSelector, { timeout: 5000 }); // Wait for navigation or response const [response] = await Promise.all([ page .waitForNavigation({ waitUntil: "domcontentloaded", timeout: 10000 }) .catch(() => null), // Don't fail if no navigation page.click(submitSelector), ]); // Check for submission success const currentUrl = page.url(); const submitted = response !== null || currentUrl !== tab.url; return { submitted, response: response ? await response.text() : null, redirectUrl: submitted ? currentUrl : undefined, errors: [], }; } catch (error) { return { submitted: false, response: null, errors: [error.message], }; } } private async handleCaptcha(page: any): Promise<void> { // Simple CAPTCHA detection and handling try { const captchaElements = await page.$$eval( '[class*="captcha"], [id*="captcha"], [src*="captcha"]', (elements: any[]) => elements.length, ); if (captchaElements > 0) { this.logger.info("CAPTCHA detected, attempting to solve"); if (this.config.enabled) { // Implement CAPTCHA solving logic here // This would integrate with services like 2captcha, AntiCaptcha, etc. this.fillerMetrics.captchasSolved++; } else { throw new Error("CAPTCHA detected but solving is disabled"); } } } catch (error) { this.logger.warn("CAPTCHA handling failed", error); } } // === INITIALIZATION HELPERS === private initializeFieldMappings(): void { // Common field mappings const commonMappings: Array<[string, FieldMapping]> = [ [ "email", { sourceField: "email", targetSelector: '[type="email"], [name*="email"], [id*="email"]', priority: 10, }, ], [ "firstName", { sourceField: "firstName", targetSelector: '[name*="first"], [id*="first"], [name*="fname"]', priority: 9, }, ], [ "lastName", { sourceField: "lastName", targetSelector: '[name*="last"], [id*="last"], [name*="lname"]', priority: 9, }, ], [ "phone", { sourceField: "phone", targetSelector: '[type="tel"], [name*="phone"], [id*="phone"]', priority: 8, }, ], [ "address", { sourceField: "address", targetSelector: '[name*="address"], [id*="address"], [name*="street"]', priority: 7, }, ], ]; for (const [key, mapping] of commonMappings) { this.fieldMappings.set(key, mapping); } // Add configured mappings for (const [source, target] of Object.entries(this.config.fieldMapping)) { this.fieldMappings.set(source, { sourceField: source, targetSelector: target, priority: 5, }); } } private initializeSmartDefaults(): void { this.smartDefaults = [ { pattern: /email/i, generator: () => `user${Date.now()}@example.com`, category: "email", }, { pattern: /first.*name|fname/i, generator: () => "John", category: "name", }, { pattern: /last.*name|lname/i, generator: () => "Doe", category: "name", }, { pattern: /phone|tel/i, generator: () => "+1-555-0123", category: "phone", }, { pattern: /zip|postal/i, generator: () => "12345", category: "postal", }, { pattern: /city/i, generator: () => "New York", category: "location", }, ]; } private loadValidationRules(): void { // Add configured validation rules for (const rule of this.config.validationRules) { this.logger.debug("Loaded validation rule", { field: rule.field, type: rule.type, required: rule.required, }); } } private async humanLikeDelay(): Promise<void> { // Random delay between 100-500ms to simulate human typing const delay = Math.random() * 400 + 100; await new Promise((resolve) => setTimeout(resolve, delay)); } }