@sconedev/ai_toolkit
Version:
Simplify AI integration in web apps with local and offline model support
207 lines (206 loc) • 9.36 kB
JavaScript
import { chat } from './chat';
// Import PDF.js the right way - named exports
import { getDocument, GlobalWorkerOptions, version } from 'pdfjs-dist';
// Set worker path with better error handling
if (typeof window !== 'undefined') {
try {
// Check if PDF.js is properly loaded
if (!getDocument || typeof getDocument !== 'function') {
console.error('PDF.js is not properly imported. The getDocument function is not available.');
}
const pdfjsVersion = version || '4.10.38';
GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjsVersion}/pdf.worker.min.mjs`;
console.log(`PDF.js worker configured with version ${pdfjsVersion}`);
}
catch (err) {
console.error('Failed to configure PDF.js worker:', err);
}
}
/**
* Splits text into manageable chunks to avoid token limits
*/
function splitTextIntoChunks(text, maxChunkSize = 4000) {
const chunks = [];
let startIndex = 0;
while (startIndex < text.length) {
let endIndex = Math.min(startIndex + maxChunkSize, text.length);
// If we're not at the end, find a good breaking point
if (endIndex < text.length) {
// Try to break at a paragraph boundary first, then sentence
const lastParagraph = text.lastIndexOf('\n\n', endIndex);
if (lastParagraph > startIndex && lastParagraph > endIndex - 200) {
endIndex = lastParagraph + 2;
}
else {
// Try to break at a sentence boundary
const lastPeriod = text.lastIndexOf('.', endIndex);
if (lastPeriod > startIndex && lastPeriod > endIndex - 100) {
endIndex = lastPeriod + 1;
}
}
}
chunks.push(text.substring(startIndex, endIndex));
startIndex = endIndex;
}
return chunks;
}
// Stub functions for Node.js methods to prevent errors in browser
export function extractTextFromPDF() {
throw new Error('extractTextFromPDF is only available in Node.js environment. Use summarizePDFBrowser in browser.');
}
export function summarizePDF() {
throw new Error('summarizePDF is only available in Node.js environment. Use summarizePDFBrowser in browser.');
}
/**
* Summarizes extracted text using AI
*/
export async function summarizeText(text, options) {
if (!text || text.trim().length === 0) {
throw new Error('No text provided for summarization');
}
const chunkSize = options?.chunkSize || 4000;
const additionalInstructions = options?.additionalInstructions || '';
const timeout = options?.timeout || 60000; // Default 60s timeout
// If text is small enough, summarize directly
if (text.length <= chunkSize) {
const messages = [
{ role: 'system', content: `You are a helpful assistant that summarizes text into bullet points. ${additionalInstructions}` },
{ role: 'user', content: `Please summarize the following text into concise bullet points:\n\n${text}` }
];
return await Promise.race([
chat(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error('Summarization timed out')), timeout))
]);
}
// Split text into chunks and summarize each chunk
const chunks = splitTextIntoChunks(text, chunkSize);
const chunkSummaries = [];
for (let i = 0; i < chunks.length; i++) {
const messages = [
{ role: 'system', content: `You are a helpful assistant that summarizes text into bullet points. This is part ${i + 1} of ${chunks.length}. ${additionalInstructions}` },
{ role: 'user', content: `Please summarize the following text (part ${i + 1} of ${chunks.length}) into concise bullet points:\n\n${chunks[i]}` }
];
try {
const chunkSummary = await Promise.race([
chat(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error(`Summarization of chunk ${i + 1} timed out`)), timeout))
]);
chunkSummaries.push(chunkSummary);
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
throw new Error(`Failed to summarize chunk ${i + 1}: ${errorMessage}`);
}
}
// If we have multiple summaries, combine them
if (chunkSummaries.length > 1) {
const combinedSummary = chunkSummaries.join('\n\n');
const messages = [
{ role: 'system', content: `You are a helpful assistant that combines multiple summaries into a cohesive bullet-point summary. ${additionalInstructions}` },
{ role: 'user', content: `Please combine these summaries into a cohesive set of bullet points, removing redundancies:\n\n${combinedSummary}` }
];
return await Promise.race([
chat(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error('Final summary combination timed out')), timeout))
]);
}
return chunkSummaries[0] || '';
}
/**
* Browser-compatible function for File objects
* Users only need to provide a file and options - no external libraries needed
*/
export async function summarizePDFBrowser(file, options) {
if (!file) {
throw new Error('No file provided');
}
if (file.type !== 'application/pdf') {
throw new Error('The file must be a PDF');
}
// Add debugging information
console.log('PDF file being processed:', {
name: file.name,
size: file.size,
type: file.type,
options
});
const maxPages = options?.maxPages || Infinity;
try {
// Get file as array buffer
console.log('Converting file to ArrayBuffer...');
const arrayBuffer = await file.arrayBuffer();
// Check if PDF.js is properly imported
if (!getDocument || typeof getDocument !== 'function') {
throw new Error('PDF.js getDocument function is not available. ' +
'This could be due to an import error or bundling issue.');
}
// Load the PDF using the named import directly
console.log('Loading PDF document...');
const loadingTask = getDocument({ data: arrayBuffer });
if (!loadingTask || typeof loadingTask.promise !== 'object') {
throw new Error('PDF.js loading task is invalid. ' +
'Check that you\'re using a compatible version of PDF.js.');
}
const pdf = await loadingTask.promise;
console.log(`PDF loaded successfully with ${pdf.numPages} pages`);
// Process pages with better tracking
let fullText = '';
const pageCount = Math.min(pdf.numPages, maxPages);
console.log(`Processing ${pageCount} pages out of ${pdf.numPages} total`);
let processedPages = 0;
let errorPages = 0;
for (let i = 1; i <= pageCount; i++) {
try {
console.log(`Processing page ${i} of ${pageCount}...`);
const page = await pdf.getPage(i);
const content = await page.getTextContent();
if (content && Array.isArray(content.items)) {
// Extract text more robustly
const pageText = content.items
.filter(item => item !== null && typeof item === 'object')
.map(item => {
if ('str' in item && typeof item.str === 'string') {
return item.str;
}
return '';
})
.join(' ');
fullText += pageText + '\n\n';
processedPages++;
}
else {
console.warn(`Page ${i} has no text content or unexpected format:`, content);
}
}
catch (pageError) {
errorPages++;
console.warn(`Error extracting text from page ${i}:`, pageError);
}
}
console.log(`PDF processing complete. Successfully processed ${processedPages} pages, errors on ${errorPages} pages.`);
if (processedPages === 0) {
throw new Error('Could not extract any text from the PDF. The document may be empty, encrypted, or contain only images.');
}
if (!fullText || fullText.trim().length === 0) {
throw new Error('Extracted text is empty. The PDF may contain only images or non-selectable text.');
}
return summarizeText(fullText, options);
}
catch (error) {
// Improved error handling with more context
console.error('PDF processing failed:', error);
// Format a more helpful error message
let errorMessage = 'PDF processing error: ';
if (error instanceof Error) {
errorMessage += error.message;
// Add special handling for common errors
if (error.message.includes('getDocument')) {
errorMessage += ' - This is likely due to PDF.js not being properly imported or initialized.';
}
}
else {
errorMessage += 'Unknown error occurred';
}
throw new Error(errorMessage);
}
}