@sconedev/ai_toolkit
Version:
Simplify AI integration in web apps with local and offline model support
193 lines (192 loc) • 9.06 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractTextFromPDF = extractTextFromPDF;
exports.summarizeText = summarizeText;
exports.summarizePDF = summarizePDF;
exports.summarizePDFBrowser = summarizePDFBrowser;
const chat_1 = require("./chat");
const fs = __importStar(require("fs"));
const pdf_parse_1 = __importDefault(require("pdf-parse"));
const isBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined';
function splitTextIntoChunks(text, maxChunkSize = 4000) {
const chunks = [];
let startIndex = 0;
while (startIndex < text.length) {
let endIndex = Math.min(startIndex + maxChunkSize, text.length);
if (endIndex < text.length) {
// Try to break at a paragraph boundary first, then sentence
const lastParagraph = text.lastIndexOf('\n\n', endIndex);
if (lastParagraph > startIndex && lastParagraph > endIndex - 200) {
endIndex = lastParagraph + 2;
}
else {
// Try to break at a sentence boundary
const lastPeriod = text.lastIndexOf('.', endIndex);
if (lastPeriod > startIndex && lastPeriod > endIndex - 100) {
endIndex = lastPeriod + 1;
}
}
}
chunks.push(text.substring(startIndex, endIndex));
startIndex = endIndex;
}
return chunks;
}
/**
* Extracts text from a PDF file in Node.js environment
*/
function extractTextFromPDF(pdfPath) {
return __awaiter(this, void 0, void 0, function* () {
if (isBrowser) {
throw new Error('extractTextFromPDF is only available in Node.js environment. Use summarizePDFBrowser in browser.');
}
let dataBuffer;
try {
if (typeof pdfPath === 'string') {
dataBuffer = fs.readFileSync(pdfPath);
}
else {
dataBuffer = pdfPath;
}
const data = yield (0, pdf_parse_1.default)(dataBuffer, {
// Add timeout to prevent hanging on corrupt files
max: 0 // No limit
});
return data.text || '';
}
catch (error) {
if (error instanceof Error) {
if (error.message.includes('password')) {
throw new Error('Cannot process password-protected PDF files');
}
throw new Error(`Failed to extract text from PDF: ${error.message}`);
}
throw new Error('Failed to extract text from PDF: Unknown error');
}
});
}
/**
* Summarizes extracted text using AI
*/
function summarizeText(text, options) {
return __awaiter(this, void 0, void 0, function* () {
if (!text || text.trim().length === 0) {
throw new Error('No text provided for summarization');
}
const chunkSize = (options === null || options === void 0 ? void 0 : options.chunkSize) || 4000;
const additionalInstructions = (options === null || options === void 0 ? void 0 : options.additionalInstructions) || '';
const timeout = (options === null || options === void 0 ? void 0 : options.timeout) || 60000; // Default 60s timeout
// If text is small enough, summarize directly
if (text.length <= chunkSize) {
const messages = [
{ role: 'system', content: `You are a helpful assistant that summarizes text into bullet points. ${additionalInstructions}` },
{ role: 'user', content: `Please summarize the following text into concise bullet points:\n\n${text}` }
];
return yield Promise.race([
(0, chat_1.chat)(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error('Summarization timed out')), timeout))
]);
}
// Split text into chunks and summarize each chunk
const chunks = splitTextIntoChunks(text, chunkSize);
const chunkSummaries = [];
for (let i = 0; i < chunks.length; i++) {
const messages = [
{ role: 'system', content: `You are a helpful assistant that summarizes text into bullet points. This is part ${i + 1} of ${chunks.length}. ${additionalInstructions}` },
{ role: 'user', content: `Please summarize the following text (part ${i + 1} of ${chunks.length}) into concise bullet points:\n\n${chunks[i]}` }
];
try {
const chunkSummary = yield Promise.race([
(0, chat_1.chat)(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error(`Summarization of chunk ${i + 1} timed out`)), timeout))
]);
chunkSummaries.push(chunkSummary);
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
throw new Error(`Failed to summarize chunk ${i + 1}: ${errorMessage}`);
}
}
// If we have multiple summaries, combine them
if (chunkSummaries.length > 1) {
const combinedSummary = chunkSummaries.join('\n\n');
const messages = [
{ role: 'system', content: `You are a helpful assistant that combines multiple summaries into a cohesive bullet-point summary. ${additionalInstructions}` },
{ role: 'user', content: `Please combine these summaries into a cohesive set of bullet points, removing redundancies:\n\n${combinedSummary}` }
];
return yield Promise.race([
(0, chat_1.chat)(messages),
new Promise((_, reject) => setTimeout(() => reject(new Error('Final summary combination timed out')), timeout))
]);
}
return chunkSummaries[0] || '';
});
}
/**
* Main function to summarize a PDF file in Node.js
*/
function summarizePDF(pdfPath, options) {
return __awaiter(this, void 0, void 0, function* () {
if (isBrowser) {
throw new Error('summarizePDF is only available in Node.js environment. Use summarizePDFBrowser in browser.');
}
const text = yield extractTextFromPDF(pdfPath);
if (!text || text.trim().length === 0) {
throw new Error('The PDF appears to be empty or contains no extractable text');
}
return summarizeText(text, options);
});
}
function summarizePDFBrowser(_file, _options) {
return __awaiter(this, void 0, void 0, function* () {
if (!isBrowser) {
throw new Error('summarizePDFBrowser is only available in browser environment. Use summarizePDF in Node.js.');
}
throw new Error('This is a stub function for Node.js. Make sure the browser version is properly loaded.');
});
}