@sconedev/ai_toolkit/dist/utils/web.js

Simplify AI integration in web apps with local and offline model support

/**
 * Extracts text content from a URL by fetching the page and parsing its content
 * @param url The URL to fetch content from
 * @param options Additional options for extraction
 * @returns The extracted text content
 */
export async function extractTextFromURL(url, options) {
    try {
        const controller = new AbortController();
        const timeoutId = setTimeout(() => controller.abort(), options?.timeout || 10000);
        const response = await fetch(url, { signal: controller.signal });
        clearTimeout(timeoutId);
        if (!response.ok) {
            throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
        }
        const html = await response.text();
        // Create a temporary DOM element to parse HTML
        const parser = new DOMParser();
        const doc = parser.parseFromString(html, 'text/html');
        // Remove script and style elements
        const scripts = doc.querySelectorAll('script, style');
        scripts.forEach(el => el.remove());
        // Extract text content
        let text = doc.body.textContent || '';
        text = text.replace(/\s+/g, ' ').trim();
        return text;
    }
    catch (error) {
        throw new Error(`Error extracting text from URL: ${error instanceof Error ? error.message : String(error)}`);
    }
}