@sammy-labs/sammy-three/dist/index.d.ts

Sammy Agent Core: AI voice agent with screen capture, memory management, and live API integration

import * as _google_genai from '@google/genai';
import { LiveConnectConfig, Part, Content, LiveServerToolCall, GoogleGenAIOptions, LiveClientToolResponse, LiveServerMessage, GoogleGenAI, Session, LiveServerToolCallCancellation, LiveServerContent, FunctionCall, FunctionResponse, FunctionDeclaration, Modality, StartSensitivity, EndSensitivity, ActivityHandling, TurnCoverage, HarmCategory, HarmBlockThreshold } from '@google/genai';
import { RefObject, ReactNode, ComponentType } from 'react';
import * as eventemitter3 from 'eventemitter3';
import eventemitter3__default, { EventEmitter } from 'eventemitter3';

/**
 * Built-in Audio Stutter Analyzer
 * Automatically collects events from audio and render systems to help debug stuttering
 */
interface AnalysisEvent {
    type: 'audio' | 'render';
    subtype: string;
    message: string;
    data: Record<string, any>;
    timestamp: number;
}
declare class AudioStutterAnalyzer {
    private static instance;
    private events;
    private isActive;
    private debugMode;
    private maxEvents;
    private constructor();
    static getInstance(): AudioStutterAnalyzer;
    setDebugMode(enabled: boolean): void;
    addEvent(type: 'audio' | 'render', subtype: string, message: string, data?: Record<string, any>): void;
    getAnalysis(): void;
    clear(): void;
    setActive(active: boolean): void;
    getEvents(): AnalysisEvent[];
    getSummary(): {
        totalEvents: number;
        underruns: number;
        longCaptures: number;
        isActive: boolean;
        debugMode: boolean;
    };
}
declare const audioStutterAnalyzer: AudioStutterAnalyzer;

/**
 * Memory Types
 * Type definitions for memory-related operations and data structures.
 * Used by memory services for search and retrieval operations.
 * Aligned with Memory API v2.0 native Supabase implementation.
 */
declare enum MemorySourceType {
    CKB = "CKB",
    DIRECT_CSM_ADD = "DIRECT_CSM_ADD",
    KNOWLEDGE_HUB = "KNOWLEDGE_HUB",
    TRAINING_SESSION = "TRAINING_SESSION",
    TRAINING_VIDEO = "TRAINING_VIDEO",
    USER_SESSION = "USER_SESSION"
}
declare enum AgentType {
    ADMIN = "ADMIN",
    USER = "USER"
}
declare enum MemoryCategory {
    AGENT_PERSONALITY = "AGENT_PERSONALITY",
    BUG_REPORT = "BUG_REPORT",
    DEPRECATED_INFO = "DEPRECATED_INFO",
    FEATURE_MEMORY = "FEATURE_MEMORY",
    FEATURE_REQUEST = "FEATURE_REQUEST",
    FEEDBACK = "FEEDBACK",
    GLOBAL_CONTEXTUAL_MEMORY = "GLOBAL_CONTEXTUAL_MEMORY",
    ONBOARDING_ISSUE = "ONBOARDING_ISSUE",
    PRODUCT_INFO = "PRODUCT_INFO",
    UPSELL_OPPORTUNITY = "UPSELL_OPPORTUNITY",
    USER_GOAL = "USER_GOAL",
    USER_PREFERENCES = "USER_PREFERENCES"
}
interface MemoryEntry {
    memory_id: string;
    organisation_id: string;
    agent_type: AgentType;
    categories: MemoryCategory[];
    created_at: string;
    global_memory: boolean;
    memory: string;
    source: MemorySourceType;
    confidence_score?: number;
    conversation_id?: string;
    sammy_three_organisation_feature_id?: string;
    created_by?: string;
    importance_score?: number;
    similarity?: number;
    updated_at?: string;
}
interface MemoryMetadata {
    agent_type: AgentType;
    global_memory: boolean;
    source: MemorySourceType;
    confidence_score?: number;
    conversation_id?: string;
    sammy_three_organisation_feature_id?: string;
    categories?: MemoryCategory[];
    importance_score?: number;
}
interface MemoryCreate {
    content: string;
    metadata: MemoryMetadata;
    infer?: boolean;
}
interface SearchRequest {
    query: string;
    feature_id?: string;
    categories?: MemoryCategory[];
    limit?: number;
    search_global_only?: boolean | null;
    similarity_threshold?: number;
}
interface SearchResponse {
    memories: MemoryEntry[];
}
interface MemoryUpdate {
    content?: string;
}
interface MessageSender {
    send(parts: Array<{
        text: string;
    }>, turnComplete: boolean): void;
}
interface MemorySearchConfig {
    agentContextPrefix?: string;
    agentOutputThreshold?: number;
    searchGlobalOnly?: boolean;
    searchLimit?: number;
    similarityThreshold?: number;
    useFeatureIdFilter?: boolean;
    userContextPrefix?: string;
    userContextSuffix?: string;
    userInputThreshold?: number;
}

interface FlushData {
    conversationId: string;
    endTime: number;
    pcmData: ArrayBuffer;
    sampleRate: number;
    speaker: 'agent' | 'user';
    startTime: number;
    totalBytesProcessed: number;
    turnNumber: number;
    timestamps?: {
        end: number;
        start: number;
    }[];
}

/**
 * Observability Types
 * Comprehensive type definitions for tracking all data flowing into and out of the Gemini Live API
 */

/**
 * Configuration for observability features
 */
interface ObservabilityConfig {
    /**
     * Whether to enable observability tracking
     */
    enabled: boolean;
    /**
     * Audio aggregation configuration
     */
    audioAggregation?: {
        /**
         * How often to flush audio buffers (ms)
         * Default: 10000 (10 seconds)
         */
        flushIntervalMs?: number;
        /**
         * Callback when audio buffer is flushed
         * Now optional - can be handled by worker instead
         */
        onFlush?: (data: FlushData) => Promise<void>;
    };
    /**
     * Worker-specific configuration
     */
    workerConfig?: {
        /**
         * How often to send batched events (ms)
         * Default: 5000 (5 seconds)
         */
        batchIntervalMs?: number;
        /**
         * Number of events to batch before sending
         * Default: 50
         */
        batchSize?: number;
    };
    /**
     * Custom callback for each event
     * Now optional - events can be sent to worker instead
     */
    callback?: ObservabilityCallback;
    /**
     * Event types to disable/filter out
     */
    disableEventTypes?: TraceEventType[];
    /**
     * Include raw audio data in events (can be large)
     */
    includeAudioData?: boolean;
    /**
     * Include image data in events (can be large)
     */
    includeImageData?: boolean;
    /**
     * Include system prompts in events (can be sensitive)
     */
    includeSystemPrompt?: boolean;
    /**
     * Log events to console for debugging
     */
    logToConsole?: boolean;
    /**
     * Additional metadata to include in all events
     */
    metadata?: Record<string, any>;
    /**
     * Use web worker for observability (recommended for high-frequency events)
     */
    useWorker?: boolean;
}
/**
 * Callback function for handling observability events
 */
type ObservabilityCallback = (event: TraceEvent) => Promise<void> | void;
/**
 * Types of events that can be traced
 */
type TraceEventType = 'agent.cleanup' | 'agent.mute' | 'agent.streaming_start' | 'agent.streaming_stop' | 'agent.unmute' | 'audio.gate_state_change' | 'audio.receive' | 'audio.recording_error' | 'audio.recording_start' | 'audio.recording_stop' | 'audio.send' | 'audio.volume_change' | 'config.set' | 'connection.close' | 'connection.error' | 'connection.open' | 'content.receive' | 'content.send' | 'context.update' | 'conversation.ready' | 'error' | 'memory.inject' | 'memory.search' | 'screen_capture.critical' | 'screen_capture.error' | 'screen_capture.send' | 'screen_capture.start' | 'screen_capture.stop' | 'session.end' | 'session.start' | 'system_prompt.set' | 'token.expired' | 'token.generated' | 'tool.call' | 'tool.register' | 'tool.response' | 'transcription.input' | 'transcription.output' | 'turn.complete' | 'url.change';
/**
 * Base structure for all trace events
 */
interface BaseTraceEvent {
    /** Unique event ID */
    id: string;
    /** Conversation ID this event belongs to */
    conversationId: string;
    /**
     * Sequence number for guaranteed ordering within a session
     * Increments with each event, starts at 0 for each session
     * Use this for reliable ordering instead of timestamp comparison
     */
    sequenceNumber: number;
    /** Session ID this event belongs to */
    sessionId: string;
    /**
     * High-resolution timestamp when event occurred
     * Uses performance.now() + sequence number for microsecond precision
     * to ensure proper ordering of rapid events like transcriptions
     */
    timestamp: Date;
    /** Event type */
    type: TraceEventType;
    /** Duration in milliseconds (for timed events) */
    durationMs?: number;
    /** Additional metadata */
    metadata?: Record<string, any>;
}
/**
 * Session lifecycle events
 */
interface SessionStartTraceEvent extends BaseTraceEvent {
    data: {
        isInternal: boolean;
        agentMode: 'admin' | 'user';
        model: string;
        flowId?: string;
    };
    type: 'session.start';
}
interface SessionEndTraceEvent extends BaseTraceEvent {
    data: {
        messageCount: number;
        reason: 'error' | 'timeout' | 'token_expired' | 'user_initiated';
        toolCallCount: number;
        totalDuration: number;
    };
    type: 'session.end';
}
/**
 * Configuration events
 */
interface ConfigTraceEvent extends BaseTraceEvent {
    data: {
        config: LiveConnectConfig;
        model: string;
    };
    type: 'config.set';
}
interface SystemPromptTraceEvent extends BaseTraceEvent {
    data: {
        characterCount: number;
        prompt: string;
        source: 'custom' | 'default' | 'service';
    };
    type: 'system_prompt.set';
}
/**
 * Memory events
 */
interface MemorySearchTraceEvent extends BaseTraceEvent {
    data: {
        limit: number;
        query: string;
        resultsFound: number;
        source: 'agent' | 'user';
        threshold: number;
    };
    type: 'memory.search';
}
interface MemoryInjectTraceEvent extends BaseTraceEvent {
    data: {
        contextType: 'live' | 'validation';
        memories: Array<{
            id: string;
            memory: string;
            score?: number;
        }>;
        source: 'agent' | 'user';
    };
    type: 'memory.inject';
}
/**
 * Content events
 */
interface ContentSendTraceEvent extends BaseTraceEvent {
    data: {
        messageType: 'mixed' | 'system' | 'text';
        parts: Part[];
        turnComplete: boolean;
        characterCount?: number;
    };
    type: 'content.send';
}
interface ContentReceiveTraceEvent extends BaseTraceEvent {
    data: {
        hasText: boolean;
        content: Content;
        hasAudio: boolean;
        characterCount?: number;
    };
    type: 'content.receive';
}
/**
 * Audio events
 */
interface AudioSendTraceEvent extends BaseTraceEvent {
    data: {
        mimeType: string;
        sizeBytes: number;
        turnNumber?: number;
        audioData?: string;
        durationMs?: number;
    };
    type: 'audio.send';
}
interface AudioReceiveTraceEvent extends BaseTraceEvent {
    data: {
        mimeType: string;
        sizeBytes: number;
        turnNumber?: number;
        audioData?: string;
    };
    type: 'audio.receive';
}
/**
 * Screen capture events
 */
interface ScreenCaptureTraceEvent extends BaseTraceEvent {
    data: {
        width: number;
        height: number;
        mimeType: string;
        sizeBytes: number;
        method: 'render' | 'screen' | 'video';
        imageData?: string;
    };
    type: 'screen_capture.send';
}
/**
 * Tool events
 */
interface ToolRegistrationTraceEvent extends BaseTraceEvent {
    data: {
        toolName: string;
        parameters?: any;
        description?: string;
    };
    type: 'tool.register';
}
interface ToolCallTraceEvent extends BaseTraceEvent {
    data: {
        parameters: any;
        toolCall: LiveServerToolCall;
        toolName: string;
    };
    type: 'tool.call';
}
interface ToolResponseTraceEvent extends BaseTraceEvent {
    data: {
        success: boolean;
        functionCallId: string;
        response: any;
        toolName: string;
        error?: string;
    };
    type: 'tool.response';
}
/**
 * Transcription events
 */
interface TranscriptionTraceEvent extends BaseTraceEvent {
    data: {
        finished: boolean;
        text: string;
        characterCount: number;
    };
    type: 'transcription.input' | 'transcription.output';
}
/**
 * Other events
 */
interface UrlChangeTraceEvent extends BaseTraceEvent {
    data: {
        from: string;
        to: string;
    };
    type: 'url.change';
}
interface ContextUpdateTraceEvent extends BaseTraceEvent {
    data: {
        details: any;
        updateType: string;
    };
    type: 'context.update';
}
interface TurnCompleteTraceEvent extends BaseTraceEvent {
    data: {
        agentMessage: string;
        turnNumber: number;
        userMessage: string;
        toolsUsed: string[];
    };
    type: 'turn.complete';
}
interface ErrorTraceEvent extends BaseTraceEvent {
    data: {
        severity: 'error' | 'fatal' | 'warning';
        context: string;
        error: string;
        stack?: string;
    };
    type: 'error';
}
interface AgentMuteTraceEvent extends BaseTraceEvent {
    data: {
        muted: boolean;
    };
    type: 'agent.mute';
}
interface AgentUnmuteTraceEvent extends BaseTraceEvent {
    data: {
        muted: boolean;
    };
    type: 'agent.unmute';
}
interface AgentStreamingStartTraceEvent extends BaseTraceEvent {
    data: {
        streamType: 'audio' | 'both' | 'screen';
    };
    type: 'agent.streaming_start';
}
interface AgentStreamingStopTraceEvent extends BaseTraceEvent {
    data: {
        streamType: 'audio' | 'both' | 'screen';
        reason?: string;
    };
    type: 'agent.streaming_stop';
}
interface AgentCleanupTraceEvent extends BaseTraceEvent {
    data: {
        reason: 'error' | 'timeout' | 'token_expired' | 'user_initiated';
    };
    type: 'agent.cleanup';
}
interface AudioRecordingStartTraceEvent extends BaseTraceEvent {
    data: {
        sampleRate: number;
        noiseGateEnabled: boolean;
        noiseSuppressionEnabled: boolean;
    };
    type: 'audio.recording_start';
}
interface AudioRecordingStopTraceEvent extends BaseTraceEvent {
    data: {
        duration?: number;
        reason?: string;
    };
    type: 'audio.recording_stop';
}
interface AudioRecordingErrorTraceEvent extends BaseTraceEvent {
    data: {
        error: string;
        errorType: 'device_error' | 'permission_denied' | 'processing_error' | 'unknown';
    };
    type: 'audio.recording_error';
}
interface AudioGateStateChangeTraceEvent extends BaseTraceEvent {
    data: {
        state: 'closed' | 'open';
        threshold: number;
        volume: number;
    };
    type: 'audio.gate_state_change';
}
interface AudioVolumeChangeTraceEvent extends BaseTraceEvent {
    data: {
        source: 'agent' | 'user';
        volume: number;
    };
    type: 'audio.volume_change';
}
interface ConnectionOpenTraceEvent extends BaseTraceEvent {
    data: {
        status: string;
        model: string;
    };
    type: 'connection.open';
}
interface ConnectionCloseTraceEvent extends BaseTraceEvent {
    data: {
        wasClean: boolean;
        code: number;
        reason: string;
    };
    type: 'connection.close';
}
interface ConnectionErrorTraceEvent extends BaseTraceEvent {
    data: {
        message: string;
        error: string;
    };
    type: 'connection.error';
}
interface TokenGeneratedTraceEvent extends BaseTraceEvent {
    data: {
        model: string;
        expiresAt?: string;
    };
    type: 'token.generated';
}
interface TokenExpiredTraceEvent extends BaseTraceEvent {
    data: {
        expiredAt: string;
    };
    type: 'token.expired';
}
interface ScreenCaptureStartTraceEvent extends BaseTraceEvent {
    data: {
        method: 'screen' | 'tab' | 'window';
    };
    type: 'screen_capture.start';
}
interface ScreenCaptureStopTraceEvent extends BaseTraceEvent {
    data: {
        duration?: number;
        reason?: string;
    };
    type: 'screen_capture.stop';
}
interface ScreenCaptureErrorTraceEvent extends BaseTraceEvent {
    data: {
        error: string;
        errorType: 'not_supported' | 'permission_denied' | 'unknown';
    };
    type: 'screen_capture.error';
}
interface ScreenCaptureCriticalTraceEvent extends BaseTraceEvent {
    data: {
        timeSinceLastCapture: number;
        audioActive: boolean;
        bypassedThrottling: boolean;
        captureMethod: 'dom' | 'placeholder';
        reason: 'agent-started-speaking' | 'agent-stopped-speaking' | 'interrupted' | 'turn-complete' | 'user-started-speaking' | 'user-stopped-speaking';
        width?: number;
        height?: number;
        sizeBytes?: number;
    };
    type: 'screen_capture.critical';
}
interface ConversationReadyTraceEvent extends BaseTraceEvent {
    data: {
        success: boolean;
        conversationId: string;
        message?: string;
    };
    type: 'conversation.ready';
}
/**
 * Union type of all trace events
 */
type TraceEvent = AgentCleanupTraceEvent | AgentMuteTraceEvent | AgentStreamingStartTraceEvent | AgentStreamingStopTraceEvent | AgentUnmuteTraceEvent | AudioGateStateChangeTraceEvent | AudioReceiveTraceEvent | AudioRecordingErrorTraceEvent | AudioRecordingStartTraceEvent | AudioRecordingStopTraceEvent | AudioSendTraceEvent | AudioVolumeChangeTraceEvent | ConfigTraceEvent | ConnectionCloseTraceEvent | ConnectionErrorTraceEvent | ConnectionOpenTraceEvent | ContentReceiveTraceEvent | ContentSendTraceEvent | ContextUpdateTraceEvent | ConversationReadyTraceEvent | ErrorTraceEvent | MemoryInjectTraceEvent | MemorySearchTraceEvent | ScreenCaptureCriticalTraceEvent | ScreenCaptureErrorTraceEvent | ScreenCaptureStartTraceEvent | ScreenCaptureStopTraceEvent | ScreenCaptureTraceEvent | SessionEndTraceEvent | SessionStartTraceEvent | SystemPromptTraceEvent | TokenExpiredTraceEvent | TokenGeneratedTraceEvent | ToolCallTraceEvent | ToolRegistrationTraceEvent | ToolResponseTraceEvent | TranscriptionTraceEvent | TurnCompleteTraceEvent | UrlChangeTraceEvent;
/**
 * Complete session trace containing all events
 */
interface SessionTrace {
    summary: {
        errorCount: number;
        eventCounts: Record<TraceEventType, number>;
        messageCount: number;
        toolCallCount: number;
        totalEvents: number;
        totalDuration?: number;
    };
    conversationId: string;
    events: TraceEvent[];
    sessionId: string;
    startTime: Date;
    endTime?: Date;
}
/**
 * Statistics for the current session
 */
interface SessionStatistics {
    audioBytesReceived: number;
    audioBytesSent: number;
    criticalRenders: number;
    duration: number;
    errors: number;
    eventCount: number;
    imageBytesSent: number;
    memoriesInjected: number;
    memoriesSearched: number;
    messagesReceived: number;
    messagesSent: number;
    sessionId: string;
    startTime: Date;
    toolCalls: number;
}

/**
 * Observability Manager
 * Central class for tracking all events during an agent session
 */

interface ObservabilityEventMap {
    statistics: (stats: SessionStatistics) => void;
    trace: (event: TraceEvent) => void;
}
declare class ObservabilityManager {
    private config;
    private events;
    private sessionTrace;
    private sessionStats;
    private sessionId;
    private conversationId;
    private startTime;
    private sequenceNumber;
    private baseTime;
    constructor(sessionId: string, conversationId: string, config: ObservabilityConfig);
    /**
     * Get the event emitter for subscribing to events
     */
    getEventEmitter(): EventEmitter<ObservabilityEventMap>;
    /**
     * Generate a high-resolution timestamp to ensure event ordering
     */
    private generateHighResTimestamp;
    /**
     * Track an event
     */
    trackEvent(event: Omit<TraceEvent, 'conversationId' | 'id' | 'sequenceNumber' | 'sessionId' | 'timestamp'>): Promise<void>;
    /**
     * Apply privacy settings to redact sensitive data
     */
    private applyPrivacySettings;
    /**
     * Update statistics based on event
     */
    private updateStatistics;
    /**
     * End the session and return final trace
     */
    endSession(reason?: 'error' | 'timeout' | 'token_expired' | 'user_initiated'): SessionTrace;
    /**
     * Get current trace
     */
    getTrace(): SessionTrace;
    /**
     * Get current statistics
     */
    getStatistics(): SessionStatistics;
    /**
     * Serialize events for export
     */
    serializeEvents(events: TraceEvent[]): string;
    /**
     * Export trace as JSON
     */
    exportAsJson(): string;
    /**
     * Generate a summary report
     */
    generateSummaryReport(): string;
}

/**
 * Centralized Context State Manager
 *
 * This class manages all contextual state for the agent including:
 * - User memories from memory searches
 * - Page metadata (URL, title, timestamp)
 * - User preferences and settings
 * - Custom contextual data
 *
 * It provides a single source of truth for context and handles
 * formatting for LLM consumption.
 */

interface PageMetadata {
    timestamp: string;
    title: string;
    url: string;
    domain?: string;
    path?: string;
}
interface UserPreferences {
    name?: string;
    preferences?: Record<string, any>;
    settings?: Record<string, any>;
}
interface ContextState {
    lastUpdated: {
        pageMetadata?: Date;
        userPreferences?: Date;
        memories?: Date;
    };
    customContext: Record<string, any>;
    memories: MemoryEntry[];
    pageMetadata: PageMetadata | null;
    userPreferences: UserPreferences | null;
}
interface ContextFormattingOptions {
    format?: 'compact' | 'detailed';
    includeCustomContext?: boolean;
    includeMemories?: boolean;
    includePageMetadata?: boolean;
    includeUserPreferences?: boolean;
    memoryLimit?: number;
}
declare class ContextStateManager {
    private state;
    private observabilityManager?;
    constructor(observabilityManager?: ObservabilityManager);
    updateMemories(memories: MemoryEntry[]): void;
    getMemories(): MemoryEntry[];
    clearMemories(): void;
    updatePageMetadata(metadata: Partial<PageMetadata>): void;
    getPageMetadata(): PageMetadata | null;
    updateUserPreferences(preferences: Partial<UserPreferences>): void;
    getUserPreferences(): UserPreferences | null;
    setCustomContext(key: string, value: any): void;
    getCustomContext(key: string): any;
    removeCustomContext(key: string): void;
    formatContext(options?: ContextFormattingOptions): string;
    formatForToolResponse(): string;
    formatPageContextOnly(options?: {
        includeTimestamp?: boolean;
        additionalInfo?: Record<string, any>;
        customContext?: string;
    }): string;
    private formatPageMetadata;
    private formatUserPreferences;
    private formatMemories;
    private formatCustomContext;
    getFullState(): ContextState;
    clearAll(): void;
    hasContext(): boolean;
    getLastUpdated(type?: 'memories' | 'pageMetadata' | 'userPreferences'): Date | null;
}

/**
 * Sammy API Caller
 * A generic HTTP client configured for the Sammy API with built-in authentication,
 * error handling, and retry logic. This client is completely route-agnostic and
 * provides standard HTTP methods (GET, POST, PUT, PATCH, DELETE).
 */

interface ApiCallOptions {
    path: string;
    skipValidation?: boolean;
    body?: unknown;
    headers?: HeadersInit;
    retries?: number;
    timeout?: number;
}
declare class SammyApiClient {
    private token;
    private baseUrl;
    private onTokenExpired?;
    private isTokenValid;
    constructor(authConfig: AuthConfig);
    /**
     * Updates the authentication configuration
     */
    updateAuthConfig(authConfig: Partial<AuthConfig>): void;
    /**
     * Validates the JWT token
     */
    private validateToken;
    /**
     * Generic HTTP request method with retry logic
     */
    private makeRequest;
    /**
     * Recursive helper method for request attempts with retry logic
     */
    private attemptRequest;
    /**
     * Helper method to extract error details from response
     */
    private getErrorDetails;
    /**
     * Utility method for delays
     */
    private delay;
    /**
     * GET request
     */
    get<T = unknown>(path: string, options?: Omit<ApiCallOptions, 'body' | 'path'>): Promise<T>;
    /**
     * POST request
     */
    post<T = unknown>(path: string, body?: unknown, options?: Omit<ApiCallOptions, 'body' | 'path'>): Promise<T>;
    /**
     * PUT request
     */
    put<T = unknown>(path: string, body?: unknown, options?: Omit<ApiCallOptions, 'body' | 'path'>): Promise<T>;
    /**
     * PATCH request
     */
    patch<T = unknown>(path: string, body?: unknown, options?: Omit<ApiCallOptions, 'body' | 'path'>): Promise<T>;
    /**
     * DELETE request
     */
    delete<T = unknown>(path: string, options?: Omit<ApiCallOptions, 'body' | 'path'>): Promise<T>;
    /**
     * Get current token validity status
     */
    getTokenValidityStatus(): boolean;
}

/**
 * Conversation Service
 * Business logic service for conversation lifecycle operations using SammyApiClient.
 * Handles conversation status updates and lifecycle management.
 *
 * API endpoints:
 * - markReady: PATCH /{conversation_id}/mark-ready - Updates conversation status to trigger memory processing
 */

interface ConversationService {
    markConversationReadyToProcess: (conversationId: string) => Promise<MarkConversationReadyResponse>;
    sendMessages: (payload: MessagePayload) => Promise<void>;
    escalateConversation: (request: EscalateConversationRequest) => Promise<EscalateConversationResponse>;
}
interface MarkConversationReadyResponse {
    success: boolean;
    conversationId?: string;
    message?: string;
}
interface MessagePayload {
    messages: Array<{
        sender: 'agent' | 'user';
        text: string;
        timestamp: string;
    }>;
    agentMode: AgentMode;
    conversationId: string;
    sammyThreeOrganisationFeatureId?: string;
    guideId?: string;
}
interface EscalateConversationRequest {
    conversation_id: string;
}
interface EscalateConversationResponse {
    success: boolean;
    message: string;
    conversation_id: string;
}

/**
 * Service interfaces for Sammy Agent
 * These define the contracts for external dependencies
 */

interface EphemeralTokenConstraints {
    config: {
        response_modalities?: string[];
        session_resumption?: Record<string, any>;
    };
    model: string;
}
interface EphemeralTokenResponse$1 {
    token: string;
    ephemeral_token?: string;
    expiresAt?: string;
}
/**
 * System prompt query
 * This is a loose type to allow users to provide any function structure they want
 */
type SystemPromptQuery = Record<string, any>;
/**
 * Schema for instantiating a session request
 */
interface InstantiateSessionRequest {
    /** The mode of the agent */
    agentMode: AgentMode;
    /** The ID of the conversation to instantiate */
    conversationId: string;
    /** Optional external user ID for the conversation */
    externalUserId?: string;
    /** Optional feature ID for filtering prompts */
    sammyThreeOrganisationFeatureId?: string;
}
/**
 * Schema for instantiating a session response
 */
interface InstantiateSessionResponse {
    /** The ID of the conversation */
    conversationId: string;
}
/**
 * Error types that services might throw
 */
declare class ServiceError extends Error {
    code: string;
    status?: number | undefined;
    details?: any | undefined;
    constructor(message: string, code: string, status?: number | undefined, details?: any | undefined);
}
declare class TokenExpiredError extends ServiceError {
    constructor(message?: string, details?: any);
}

/**
 * Core Services
 * Business logic services that use the SammyApiCaller for specific API operations.
 *
 * Services included:
 * - generateEphemeralToken: Generates ephemeral tokens for live API connections
 * - getSystemPrompt: Retrieves system prompts based on agent mode and context
 */

interface CoreService {
    generateEphemeralToken: (constraints: EphemeralTokenConstraints) => Promise<EphemeralTokenResponse>;
    getSystemPrompt: (query: SystemPromptQuery) => Promise<{
        systemPrompt: string;
    }>;
    instantiateSession: (instantiateSessionRequest: InstantiateSessionRequest) => Promise<InstantiateSessionResponse>;
}
/**
 * Creates the ephemeral token service
 */
interface EphemeralTokenResponse {
    token: string;
    ephemeral_token?: string;
    expiresAt?: string;
}

/**
 * Memory Service
 * Business logic service for memory search and retrieval operations using SammyApiClient.
 * Provides memory-specific functionality for search and getAllMemories operations.
 * Configured to work with the native Supabase backend API v2.0.
 */

interface MemoryService {
    getAll: (sammyThreeOrganisationFeatureId: string | null, retryOptions?: {
        retryOnEmpty?: boolean;
    }) => Promise<MemoryEntry[]>;
    search: (query: string, options: Omit<SearchRequest, 'query'>) => Promise<MemoryEntry[]>;
}
/**
 * Creates all memory services using the provided API caller
 */
declare const createMemoryServices: (apiCaller: SammyApiClient) => MemoryService;

/**
 * Guides Service
 * Business logic service for walkthrough guide operations using SammyApiClient.
 * Handles guide validation and retrieval for walkthrough functionality.
 *
 * API endpoints:
 * - fetchGuide: GET /guides/{id} - Fetches and validates a single guide by ID
 * - getUserGuides: GET /guides - Lists user's guides with completion status
 */

interface GuidesService {
    fetchGuide: (guideId: string) => Promise<SammyGuide | null>;
    getUserGuides: () => Promise<SammyGuide[]>;
}
interface SammyGuide {
    guideId: string;
    isCompleted: boolean;
    organisationId: string;
    prompt: string;
    title: string;
}

interface StartOptions {
    agentMode: AgentMode;
    guideId?: string;
    sammyThreeOrganisationFeatureId?: string;
}
declare class SammyAgentCore<TEventMap extends Record<string, (...args: any[]) => any> = BaseToolEventMap> {
    private config;
    private callbacks;
    private tools;
    private updateAgentState;
    private sammyApiClient;
    private client;
    private coreServices;
    private memoryServices;
    private conversationServices;
    guidesServices: GuidesService;
    private audioRecorder;
    private audioStreamer;
    private screenCapture;
    private toolManager;
    private contextStateManager;
    private contextMemoryManager;
    private session;
    private isConnected;
    private currentToken;
    muted: boolean;
    isStreaming: boolean;
    agentVolume: number;
    private outputTranscriptBuffer;
    private inputTranscriptBuffer;
    constructor(options: {
        callbacks: SammyAgentCallbacks;
        updateAgentState: UpdateAgentState;
        config: SammyAgentConfig;
        tools?: ToolDefinition<TEventMap>[];
    });
    /**
     * Update observability manager in context management classes
     * Called after the observable client is created
     */
    private updateObservabilityInContextManagers;
    /**
     * Update page metadata in context state
     */
    updatePageContext(metadata: {
        path?: string;
        title?: string;
        url?: string;
        domain?: string;
    }): void;
    /**
     * Update user preferences in context state
     */
    updateUserContext(preferences: {
        name?: string;
        preferences?: Record<string, unknown>;
        settings?: Record<string, unknown>;
    }): void;
    /**
     * Get the context state manager for direct access
     */
    getContextStateManager(): ContextStateManager;
    /**
     * Register the context tool with the tool manager
     */
    private registerContextTool;
    start(options: StartOptions): Promise<AgentSession>;
    destroy(): Promise<void>;
    sendMessage(message: string): void;
    setMuted(muted: boolean): void;
    getToolEventEmitter(): TypedEventEmitter<TEventMap>;
    /**
     * Get current observability trace
     */
    getObservabilityTrace(): SessionTrace | null;
    /**
     * Export observability trace as JSON
     */
    exportObservabilityTrace(): string | null;
    /**
     * Generate observability summary report
     */
    generateObservabilitySummary(): string | null;
    /**
     * Get current session statistics
     */
    getSessionStatistics(): SessionStatistics | null;
    /**
     * Get observability event emitter
     */
    getObservabilityEvents(): eventemitter3.EventEmitter<any, any> | null;
    private setupAudioStreaming;
    private setupEventHandlers;
    private handleOpen;
    private handleClose;
    private handleError;
    private handleAudio;
    private handleInterrupted;
    private handleOutputTranscription;
    private handleInputTranscription;
    private handleTurnComplete;
    private handleToolCall;
    private startAudioRecording;
    private handleAudioData;
    private sendPendingTranscriptions;
    private arrayBufferToBase64;
}

/**
 * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

type ClientContentLog = {
    turns: Part[];
    turnComplete: boolean;
};
/**
 * the options to initiate the client, ensure apiKey is required
 */
type LiveClientOptions = GoogleGenAIOptions & {
    apiKey: string;
};
/** log types */
type StreamingLog = {
    message: ClientContentLog | LiveClientToolResponse | Omit<LiveServerMessage, 'data' | 'text'> | string;
    date: Date;
    type: string;
    area?: string;
    latencyMs?: number;
    count?: number;
};

/**
 * Event types that can be emitted by the MultimodalLiveClient.
 * Each event corresponds to a specific message from GenAI or client state change.
 */
interface LiveClientEventTypes {
    inputtranscription: (transcription: {
        finished: boolean;
        text: string;
    }) => void;
    outputtranscription: (transcription: {
        finished: boolean;
        text: string;
    }) => void;
    toolcallcancellation: (toolcallCancellation: LiveServerToolCallCancellation) => void;
    audio: (data: ArrayBuffer) => void;
    close: (event: CloseEvent) => void;
    content: (data: LiveServerContent) => void;
    error: (error: ErrorEvent) => void;
    interrupted: () => void;
    log: (log: StreamingLog) => void;
    open: () => void;
    setupcomplete: () => void;
    toolcall: (toolCall: LiveServerToolCall) => void;
    turncomplete: () => void;
}
/**
 * A event-emitting class that manages the connection to the websocket and emits
 * events to the rest of the application.
 * If you dont want to use react you can still use this.
 */
declare class GenAILiveClient extends EventEmitter<LiveClientEventTypes> {
    private _model;
    private _session;
    private _status;
    protected client: GoogleGenAI;
    protected config: LiveConnectConfig | null;
    constructor(options: LiveClientOptions);
    protected log(type: string, message: StreamingLog['message'], metadata?: {
        area?: string;
        latencyMs?: number;
    }): void;
    protected onclose(e: CloseEvent): void;
    protected onerror(e: ErrorEvent): void;
    protected onmessage(message: LiveServerMessage): Promise<void>;
    protected onopen(): void;
    connect(model: string, config: LiveConnectConfig): Promise<boolean>;
    disconnect(): boolean;
    getConfig(): {
        httpOptions?: _google_genai.HttpOptions;
        abortSignal?: AbortSignal;
        generationConfig?: _google_genai.GenerationConfig;
        responseModalities?: _google_genai.Modality[];
        temperature?: number;
        topP?: number;
        topK?: number;
        maxOutputTokens?: number;
        mediaResolution?: _google_genai.MediaResolution;
        seed?: number;
        speechConfig?: _google_genai.SpeechConfig;
        enableAffectiveDialog?: boolean;
        systemInstruction?: _google_genai.ContentUnion;
        tools?: _google_genai.ToolListUnion;
        sessionResumption?: _google_genai.SessionResumptionConfig;
        inputAudioTranscription?: _google_genai.AudioTranscriptionConfig;
        outputAudioTranscription?: _google_genai.AudioTranscriptionConfig;
        realtimeInputConfig?: _google_genai.RealtimeInputConfig;
        contextWindowCompression?: _google_genai.ContextWindowCompressionConfig;
        proactivity?: _google_genai.ProactivityConfig;
    };
    /**
     * send normal content parts such as { text }
     */
    send(parts: Part | Part[], turnComplete?: boolean): void;
    /**
     * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
     */
    sendRealtimeInput(chunks: Array<{
        data: string;
        mimeType: string;
    }>): void;
    /**
     *  send a response to a function call and provide the id of the functions you are responding to
     */
    sendToolResponse(toolResponse: LiveClientToolResponse): void;
    get model(): string | null;
    get session(): Session | null;
    get status(): "connected" | "connecting" | "disconnected";
}

type CaptureMethod = 'render' | 'video';
interface ScreenCaptureConfig {
    /**
     * Method to capture screen content
     * - 'screen': Uses getDisplayMedia() to capture screen/window
     * - 'render': Uses modern-screenshot to capture DOM elements
     */
    method: CaptureMethod;
    /**
     * Maximum dimensions for captured images
     */
    maxWidth?: number;
    /**
     * DOM change detection settings (only for 'render' method)
     */
    domChangeDetection?: {
        /**
         * Debounce time after DOM mutations stop before capturing (ms)
         */
        debounceMs?: number;
        /**
         * MutationObserver configuration
         */
        observerConfig?: MutationObserverInit;
    };
    /**
     * Automatically download captured images to local storage
     * Set to false to prevent automatic downloads (useful for S3 integration later)
     */
    autoDownload?: boolean;
    /**
     * How often to check for frames (ms)
     * For 'render' method, this is the fallback interval when no DOM changes detected
     */
    checkInterval?: number;
    debugAudioPerformance?: boolean;
    /**
     * Enable debug logging
     */
    debugLogs?: boolean;
    /**
     * Enable audio-aware adaptive capture settings
     * When true, reduces capture frequency and quality during audio playback
     * to prevent audio stuttering. Default: true
     */
    enableAudioAdaptation?: boolean;
    /**
     * JPEG quality (0.0 to 1.0)
     */
    jpegQuality?: number;
    maxHeight?: number;
    /**
     * Maximum interval between frames (ms)
     */
    maxInterval?: number;
    /**
     * Minimum interval between frames (ms)
     */
    minInterval?: number;
    /**
     * Scope of capture. Either entire document or context wrapped content
     */
    scope?: 'context' | 'document';
    /**
     * Enable smart deduplication using image hashing
     */
    useHashing?: boolean;
    /**
     * Number of worker threads to use (only used if workerUrl is provided)
     * Default: 2
     */
    workerNumber?: number;
    /**
     * Worker URL for modern-screenshot (only for 'render' method)
     * When provided, captures will be processed in web workers for better performance
     * Example for Vite: import workerUrl from 'modern-screenshot/worker?url'
     */
    workerUrl?: string;
}
interface ScreenCaptureRefs {
    videoRef: RefObject<HTMLVideoElement | null>;
    canvasRef: RefObject<HTMLCanvasElement | null>;
    contextElementRef: RefObject<HTMLDivElement | null>;
}
interface ScreenCapture extends ScreenCaptureRefs {
    setVideoStream: (stream: MediaStream | null) => void;
    videoStream: MediaStream | null;
    isStreaming: boolean;
    startStreaming: (client: GenAILiveClient) => Promise<MediaStream | void>;
    stopStreaming: () => void;
    stream: MediaStream | null;
}
interface ScreenCaptureCallbacks {
    startStreaming: (client: GenAILiveClient) => Promise<MediaStream | void>;
    stopStreaming: () => void;
}

/**
 * Core types for the Sammy Agent
 */

interface AgentState {
    activeSession: AgentSession | null;
    agentVolume: number;
    error: Error | null;
    isConnected: boolean;
    isConnecting: boolean;
    isStreaming: boolean;
    userVolume: number;
}
interface UpdateAgentState {
    (variable: keyof AgentState, value: AgentState[keyof AgentState]): void;
}
declare enum AgentMode {
    ADMIN = "ADMIN",
    SAMMY = "SAMMY",
    USER = "USER"
}
interface AgentSession {
    agentMode: AgentMode;
    conversationId: string;
    sessionId: string;
    startedAt: Date;
    sammyThreeOrganisationFeatureId?: string;
    guideId?: string;
}
interface AuthConfig {
    token: string;
    baseUrl?: string;
    onTokenExpired?: () => void;
}
interface SammyAgentCallbacks {
    onConnectionStateChange?: (connected: boolean) => void;
    onError?: (error: unknown) => void;
    onMemoryUpdate?: (result: unknown) => void;
    onObservabilityEvent?: (event: TraceEvent) => void;
    onSessionTraceComplete?: (trace: SessionTrace) => void;
    onTokenExpired?: () => void;
    onToolCall?: (tool: LiveServerToolCall) => void;
    onTurnComplete?: (summary: {
        agent: string;
        user: string;
    }) => void;
}
interface SammyAgentConfigProps {
    auth: AuthConfig;
    captureConfig?: {
        frameRate?: number;
        quality?: number;
    };
    captureMethod?: CaptureMethod;
    debugAudioPerformance?: boolean;
    debugLogs?: boolean;
    defaultVoice?: string;
    model?: string;
    observability?: ObservabilityConfig;
}
interface SammyAgentConfig extends SammyAgentConfigProps {
    screenCaptureCallbacks: ScreenCaptureCallbacks;
}

declare enum ToolName {
    END_SESSION = "endSession",
    GET_CONTEXT = "getContext",
    ESCALATE = "escalate"
}
interface BaseToolEventMap extends Record<string, (...args: any[]) => any> {
    [ToolName.END_SESSION]: () => void;
    [ToolName.ESCALATE]: (data: {
        reason: string;
        context?: string;
        apiResponse?: any;
        error?: string;
    }) => void;
}
interface ToolEventMap<_TCustomEvents = {}> extends BaseToolEventMap {
}
type ExtendedToolEventMap<TCustomEvents = {}> = BaseToolEventMap & TCustomEvents;
interface TypedEventEmitter<TEventMap extends Record<string, (...args: unknown[]) => unknown>> {
    emit<K extends keyof TEventMap>(event: K, ...args: Parameters<TEventMap[K]>): boolean;
    off<K extends keyof TEventMap>(event: K, listener: TEventMap[K]): this;
    on<K extends keyof TEventMap>(event: K, listener: TEventMap[K]): this;
}
type TypedFunctionDeclaration = Omit<FunctionDeclaration, 'name'> & {
    name: ToolName;
};
type ToolCategory = 'action' | 'context';
declare const ToolCategories: {
    readonly ACTION: "action";
    readonly CONTEXT: "context";
};
interface ToolServices {
    conversationServices: ConversationService;
    coreServices: CoreService;
    memoryServices: MemoryService;
}
interface ToolContext<TEventMap extends Record<string, (...args: unknown[]) => unknown> = BaseToolEventMap> {
    emit: <K extends keyof TEventMap>(event: K, ...args: Parameters<TEventMap[K]>) => void;
    getState: () => AgentSession;
    services: ToolServices;
}
interface ToolDefinition<TEventMap extends Record<string, (...args: any[]) => any> = BaseToolEventMap> {
    handler: (fc: FunctionCall, context: ToolContext<TEventMap>) => Promise<FunctionResponse>;
    category: ToolCategory;
    declaration: TypedFunctionDeclaration;
}

/**
 * Configuration settings for the Sammy Agent
 *
 * This file contains the base configuration for the Gemini Live API connection,
 * including audio settings, activity detection, safety settings, and tool declarations.
 * The configuration is optimized for natural conversation flow with proper turn-taking
 * and reduced interruptions.
 */

/**
 * Base configuration for customer success agent sessions
 * Optimized for natural conversation flow with proper turn-taking
 */
declare const getAgentConfig: (functionDeclarations: TypedFunctionDeclaration[], systemPrompt: string) => {
    responseModalities: Modality[];
    temperature: number;
    topP: number;
    topK: number;
    maxOutputTokens: number;
    seed: number;
    speechConfig: {
        voiceConfig: {
            prebuiltVoiceConfig: {
                voiceName: string;
            };
        };
        languageCode: string;
    };
    realtimeInputConfig: {
        automaticActivityDetection: {
            disabled: boolean;
            startOfSpeechSensitivity: StartSensitivity;
            endOfSpeechSensitivity: EndSensitivity;
            prefixPaddingMs: number;
            silenceDurationMs: number;
        };
        activityHandling: ActivityHandling;
        turnCoverage: TurnCoverage;
    };
    tools: {
        functionDeclarations: TypedFunctionDeclaration[];
    }[];
    systemInstruction: {
        parts: {
            text: string;
        }[];
    };
    inputAudioTranscription: boolean;
    outputAudioTranscription: boolean;
    safetySettings: {
        category: HarmCategory;
        threshold: HarmBlockThreshold;
    }[];
};

/**
 * Hook for updating context state based on page changes
 *
 * This hook returns a function that should be called from within
 * the SammyAgentProvider where agentCoreRef is available.
 *
 * Usage:
 * const contextUpdater = useContextUpdater();
 * // Inside provider: contextUpdater(agentCoreRef.current);
 */

interface UseContextUpdaterOptions {
    includeDomain?: boolean;
    includePath?: boolean;
    includeTitle?: boolean;
    updateInterval?: number;
}
declare const useContextUpdater: (options?: UseContextUpdaterOptions) => {
    startMonitoring: (agent: SammyAgentCore | null) => void;
    stopMonitoring: () => void;
    updateUserContext: (userData: {
        name?: string;
        preferences?: Record<string, any>;
        settings?: Record<string, any>;
    }) => void;
    setCustomContext: (key: string, value: any) => void;
};

/**
 * Hook to provide always-on guide data with automatic fetching
 * Provides guide data and state management for walkthrough functionality
 */

interface UseGuidesProps {
    authConfig?: AuthConfig;
    walkthroughId?: string;
}
interface UseGuidesReturn {
    currentGuide: SammyGuide | null;
    guideError: Error | null;
    isLoadingGuide: boolean;
    isLoadingUserGuides: boolean;
    markGuideAsCompleted: (guideId: string) => void;
    refreshGuide: () => void;
    refreshUserGuides: () => void;
    userGuides: SammyGuide[];
    userGuidesError: Error | null;
}
declare function useGuides({ authConfig, walkthroughId, }: UseGuidesProps): UseGuidesReturn;

/**
 * Hook to manage guides functionality without requiring a separate context.
 * Combines guide data management, URL parameter detection, and walkthrough starting.
 */

interface UseGuidesManagerOptions {
    onStartAgent?: (options: {
        guideId: string;
        agentMode: AgentMode;
    }) => Promise<boolean>;
    authConfig?: AuthConfig;
    autoStartFromURL?: boolean;
    debug?: boolean;
    enabled?: boolean;
    onWalkthroughStart?: (guideId: string) => void;
    queryParamName?: string;
}
interface GuidesState {
    checkForGuideInUrl: () => Promise<boolean>;
    currentGuide: SammyGuide | null;
    guideError: Error | null;
    isLoadingGuide: boolean;
    isLoadingUserGuides: boolean;
    markGuideAsCompleted: (guideId: string) => void;
    refreshGuide: () => void;
    refreshUserGuides: () => void;
    userGuides: SammyGuide[];
    userGuidesError: Error | null;
    cleanupUrlParams: () => void;
    detectedWalkthroughId: string | null;
    startWalkthrough: (guideId: string) => Promise<boolean>;
}
declare function useGuidesManager({ authConfig, autoStartFromURL, onStartAgent, onWalkthroughStart, queryParamName, debug, enabled, }: UseGuidesManagerOptions): GuidesState | null;

/**
 * Hook to handle guides-related query parameters
 * Detects ?walkthrough=guide-id in URL and triggers guide start
 */
interface UseGuidesQueryParamsOptions {
    onGuideDetected?: (guideId: string) => Promise<boolean>;
    checkQueryOnMount?: boolean;
    debug?: boolean;
    queryParamName?: string;
}
interface UseGuidesQueryParamsReturn {
    checkQueryParameters: () => Promise<boolean>;
    cleanupQueryParams: () => void;
}
/**
 * Hook to detect and handle guide walkthrough query parameters
 *
 * Example URLs:
 * - https://app.com?walkthrough=guide-123
 * - https://app.com/page?walkthrough=onboarding-guide
 */
declare function useGuidesQueryParams({ checkQueryOnMount, onGuideDetected, queryParamName, debug, }?: UseGuidesQueryParamsOptions): UseGuidesQueryParamsReturn;

interface UseRenderCaptureProps {
    canvasRef: RefObject<HTMLCanvasElement | null>;
    config: ScreenCaptureConfig;
    contextElementRef: RefObject<HTMLDivElement | null>;
}
interface CriticalRenderEvent {
    reason: 'agent-started-speaking' | 'agent-stopped-speaking' | 'interrupted' | 'turn-complete' | 'user-started-speaking' | 'user-stopped-speaking';
    bypassAudioThrottling: true;
    timestamp: number;
}
interface UseRenderCaptureReturn {
    isCapturing: boolean;
    startRenderCapture: (client: GenAILiveClient) => void;
    stopRenderCapture: () => void;
    triggerCriticalRender: (event: CriticalRenderEvent) => void;
}

declare function useRenderCapture({ contextElementRef, canvasRef, config, }: UseRenderCaptureProps): UseRenderCaptureReturn;

interface UseScreenCaptureProps {
    config: ScreenCaptureConfig;
}
declare function useScreenCapture({ config, }: UseScreenCaptureProps): ScreenCapture;

/**
 * useVideoCapture Hook
 *   Captures video frames from screen recording and sends them to the AI client
 *   Returns callbacks to start/