UNPKG

make-abstract

Version:

A CLI tool for making abstracts using AI and Zotero

1,305 lines (1,274 loc) 66.6 kB
#!/usr/bin/env node // src/index.ts import { Command } from "commander"; // src/config.ts import Conf from "conf"; // src/models.ts var MODEL_PRICING = { openai: { // GPT-4.1 Series (Released April 2025) "gpt-4.1": { input: 2, output: 8 }, "gpt-4.1-mini": { input: 0.4, output: 1.6 }, "gpt-4.1-nano": { input: 0.1, output: 0.4 }, // GPT-4o Series (Current flagship) "gpt-4o": { input: 2.5, output: 10 }, "gpt-4o-mini": { input: 0.15, output: 0.6 }, "gpt-4o-2024-11-20": { input: 2.5, output: 10 }, "gpt-4o-2024-08-06": { input: 2.5, output: 10 }, "gpt-4o-2024-05-13": { input: 2.5, output: 10 }, "gpt-4o-realtime": { input: 2.5, output: 10 }, "gpt-4o-audio-preview": { input: 2.5, output: 10 }, "gpt-4o-mini-audio-preview": { input: 0.15, output: 0.6 }, "gpt-4o-search-preview": { input: 2.5, output: 10 }, "gpt-4o-mini-search-preview": { input: 0.15, output: 0.6 }, "chatgpt-4o-latest": { input: 5, output: 15 }, // March 2025 version // o1 Reasoning Series "o1": { input: 15, output: 60 }, "o1-pro": { input: 60, output: 240 }, "o1-preview": { input: 15, output: 60 }, "o1-mini": { input: 3, output: 12 }, // o3 Reasoning Series (Latest) "o3": { input: 20, output: 80 }, "o3-mini": { input: 1, output: 4 }, "o3-pro": { input: 100, output: 400 }, // o4 Series (Experimental) "o4-mini": { input: 0.5, output: 2 }, // GPT-4.5 Preview "gpt-4.5-preview": { input: 3, output: 12 }, // GPT-4 Legacy Models "gpt-4": { input: 30, output: 60 }, "gpt-4-0613": { input: 30, output: 60 }, "gpt-4-0314": { input: 30, output: 60 }, "gpt-4-32k": { input: 60, output: 120 }, "gpt-4-32k-0613": { input: 60, output: 120 }, // GPT-4 Turbo Series "gpt-4-turbo": { input: 10, output: 30 }, "gpt-4-turbo-2024-04-09": { input: 10, output: 30 }, "gpt-4-turbo-preview": { input: 10, output: 30 }, "gpt-4-1106-preview": { input: 10, output: 30 }, "gpt-4-0125-preview": { input: 10, output: 30 }, // GPT-3.5 Series "gpt-3.5-turbo": { input: 0.5, output: 1.5 }, "gpt-3.5-turbo-0125": { input: 0.5, output: 1.5 }, "gpt-3.5-turbo-1106": { input: 1, output: 2 }, "gpt-3.5-turbo-16k": { input: 3, output: 4 }, "gpt-3.5-turbo-instruct": { input: 1.5, output: 2 }, // Embeddings "text-embedding-3-large": { input: 0.13, output: 0 }, "text-embedding-3-small": { input: 0.02, output: 0 }, "text-embedding-ada-002": { input: 0.1, output: 0 }, // Default for unspecified OpenAI models default: { input: 2.5, output: 10 } }, anthropic: { // Claude 4 Series (Latest) "claude-4-opus": { input: 20, output: 100 }, "claude-4-sonnet": { input: 5, output: 25 }, "claude-4-opus-extended-thinking": { input: 30, output: 150 }, "claude-4-sonnet-extended-thinking": { input: 7.5, output: 37.5 }, // Claude 3.7 Series "claude-3-7-sonnet": { input: 4, output: 20 }, "claude-3.7-sonnet": { input: 4, output: 20 }, "claude-3.7-sonnet-extended-thinking": { input: 6, output: 30 }, // Claude 3.5 Series (Current stable) "claude-3-5-sonnet-v2": { input: 3, output: 15 }, "claude-3-5-sonnet-20241022": { input: 3, output: 15 }, "claude-3-5-sonnet-20240620": { input: 3, output: 15 }, "claude-3-5-haiku-20241022": { input: 1, output: 5 }, "claude-3-5-sonnet": { input: 3, output: 15 }, "claude-3-5-haiku": { input: 1, output: 5 }, // Claude 3 Series "claude-3-opus-20240229": { input: 15, output: 75 }, "claude-3-sonnet-20240229": { input: 3, output: 15 }, "claude-3-haiku-20240307": { input: 0.25, output: 1.25 }, "claude-3-opus": { input: 15, output: 75 }, "claude-3-sonnet": { input: 3, output: 15 }, "claude-3-haiku": { input: 0.25, output: 1.25 }, // Claude 2 Series "claude-2.1": { input: 8, output: 24 }, "claude-2.0": { input: 8, output: 24 }, "claude-instant-1.2": { input: 0.8, output: 2.4 }, // Default for unspecified Anthropic models default: { input: 3, output: 15 } }, gemini: { // Gemini 2.5 Series (Latest - Thinking Models) "gemini-2.5-pro": { input: 1.25, output: 10 }, // <=200k tokens, 2.50/15.00 for >200k "gemini-2.5-pro-preview-06-05": { input: 1.25, output: 10 }, "gemini-2.5-pro-preview-05-06": { input: 1.25, output: 10 }, "gemini-2.5-pro-exp-03-25": { input: 1.25, output: 10 }, "gemini-2.5-flash": { input: 0.3, output: 2.5 }, // Reasoning hybrid model "gemini-2.5-flash-preview-05-20": { input: 0.3, output: 2.5 }, "gemini-2.5-flash-lite-preview-06-17": { input: 0.1, output: 0.4 }, // Gemini 2.5 Audio Models "gemini-2.5-flash-preview-native-audio-dialog": { input: 0.5, output: 2 }, // text, 3.00/12.00 for audio "gemini-2.5-flash-exp-native-audio-thinking-dialog": { input: 0.5, output: 2 }, "gemini-2.5-flash-preview-tts": { input: 0.5, output: 10 }, "gemini-2.5-pro-preview-tts": { input: 1, output: 20 }, // Gemini 2.0 Series "gemini-2.0-flash": { input: 0.1, output: 0.4 }, // text/image/video, 0.70 for audio input "gemini-2.0-flash-exp": { input: 0.1, output: 0.4 }, "gemini-2.0-flash-experimental": { input: 0.1, output: 0.4 }, "gemini-2.0-flash-lite": { input: 0.075, output: 0.3 }, "gemini-2.0-flash-lite-001": { input: 0.075, output: 0.3 }, "gemini-2.0-flash-thinking-exp": { input: 0.1, output: 0.4 }, "gemini-2.0-flash-preview-image-generation": { input: 0.1, output: 0.4 }, "gemini-2.0-flash-live-001": { input: 0.1, output: 0.4 }, "gemini-2.0-pro-experimental": { input: 1.25, output: 5 }, // Gemini 1.5 Series (Stable) "gemini-1.5-pro": { input: 1.25, output: 5 }, // <=128k tokens, 2.50/10.00 for >128k "gemini-1.5-pro-002": { input: 1.25, output: 5 }, "gemini-1.5-pro-001": { input: 1.25, output: 5 }, "gemini-1.5-pro-latest": { input: 1.25, output: 5 }, "gemini-1.5-flash": { input: 0.075, output: 0.3 }, // <=128k tokens, 0.15/0.60 for >128k "gemini-1.5-flash-002": { input: 0.075, output: 0.3 }, "gemini-1.5-flash-001": { input: 0.075, output: 0.3 }, "gemini-1.5-flash-latest": { input: 0.075, output: 0.3 }, "gemini-1.5-flash-8b": { input: 0.0375, output: 0.15 }, // <=128k tokens, 0.075/0.30 for >128k "gemini-1.5-flash-8b-001": { input: 0.0375, output: 0.15 }, "gemini-1.5-flash-8b-latest": { input: 0.0375, output: 0.15 }, // Gemini 1.0 Series (Legacy) "gemini-1.0-ultra": { input: 2, output: 6 }, "gemini-1.0-pro": { input: 0.5, output: 1.5 }, "gemini-pro": { input: 0.5, output: 1.5 }, "gemini-pro-vision": { input: 0.5, output: 1.5 }, // Gemma Open Models (Free on AI Studio) "gemma-3-27b": { input: 0, output: 0 }, "gemma-3-12b": { input: 0, output: 0 }, "gemma-3-4b": { input: 0, output: 0 }, "gemma-3-1b": { input: 0, output: 0 }, "gemma-3n-4b": { input: 0, output: 0 }, "gemma-2-27b": { input: 0, output: 0 }, "gemma-2-9b": { input: 0, output: 0 }, // Embeddings "text-embedding-004": { input: 0, output: 0 }, // Free on AI Studio "gemini-embedding-exp": { input: 0, output: 0 }, "embedding-001": { input: 0, output: 0 }, // Default for unspecified Gemini models default: { input: 0.075, output: 0.3 } }, groq: { "llama-4-scout-17b-16e-instruct": { input: 0.11, output: 0.34 }, "llama-4-maverick-17b-128e-instruct": { input: 0.2, output: 0.6 }, "llama-3.3-70b-versatile": { input: 0.59, output: 0.79 }, "llama-3.1-70b-versatile": { input: 0.59, output: 0.79 }, "llama-3.1-8b-instant": { input: 0.05, output: 0.08 }, "llama-3-70b-8192": { input: 0.59, output: 0.79 }, "llama-3-8b-8192": { input: 0.05, output: 0.08 }, "mixtral-8x7b-32768": { input: 0.27, output: 0.27 }, "gemma-7b-it": { input: 0.1, output: 0.1 }, "gemma2-9b-it": { input: 0.2, output: 0.2 }, "deepseek-r1-distill-llama-70b": { input: 0.75, output: 0.99 }, // Default for unspecified Groq models default: { input: 0.27, output: 0.27 } }, deepseek: { "deepseek-chat": { input: 0.14, output: 0.28 }, // Current promotional pricing (50% off until Feb 2025) "deepseek-v3": { input: 0.14, output: 0.28 }, "deepseek-v3-0324": { input: 0.14, output: 0.28 }, "deepseek-reasoner": { input: 0.55, output: 2.19 }, // Full R1 pricing (cache miss) "deepseek-r1": { input: 0.55, output: 2.19 }, "deepseek-r1-0120": { input: 0.55, output: 2.19 }, "deepseek-r1-0528": { input: 0.55, output: 2.19 }, "deepseek-r1-lite-preview": { input: 0.3, output: 1.2 }, "deepseek-prover-v2": { input: 0.14, output: 0.28 }, "deepseek-coder": { input: 0.14, output: 0.28 }, "deepseek-coder-v2": { input: 0.14, output: 0.28 }, "deepseek-v2.5": { input: 0.14, output: 0.28 }, "deepseek-v2": { input: 0.14, output: 0.28 }, // Default for unspecified DeepSeek models default: { input: 0.14, output: 0.28 } }, cerebras: { "llama-3.3-70b": { input: 0.6, output: 0.6 }, "llama-3.1-70b": { input: 0.6, output: 0.6 }, "llama-3.1-8b": { input: 0.1, output: 0.1 }, "llama-3-70b": { input: 0.6, output: 0.6 }, "llama-3-8b": { input: 0.1, output: 0.1 }, "llama3.1-8b": { input: 0.1, output: 0.1 }, "llama3.1-70b": { input: 0.6, output: 0.6 }, // Default for unspecified Cerebras models default: { input: 0.6, output: 0.6 } }, mistral: { "mistral-large-latest": { input: 2, output: 6 }, "mistral-large-2411": { input: 2, output: 6 }, "mistral-small-3.1-2503": { input: 1, output: 3 }, "mistral-small-latest": { input: 1, output: 3 }, "codestral-2501": { input: 1, output: 3 }, "codestral-latest": { input: 1, output: 3 }, "mistral-nemo": { input: 0.3, output: 0.3 }, "mistral-ocr-2505": { input: 2, output: 6 }, "mistral-medium-latest": { input: 2.7, output: 8.1 }, "mixtral-8x7b-instruct": { input: 0.7, output: 0.7 }, "mixtral-8x22b-instruct": { input: 2, output: 6 }, "mistral-7b-instruct": { input: 0.25, output: 0.25 }, "pixtral-12b-2409": { input: 0.15, output: 0.15 }, // Default for unspecified Mistral models default: { input: 2, output: 6 } }, xai: { "grok-3-beta": { input: 2, output: 10 }, "grok-3-mini-beta": { input: 0.5, output: 2.5 }, "grok-3": { input: 2, output: 10 }, "grok-3-mini": { input: 0.5, output: 2.5 }, "grok-3-reasoning": { input: 3, output: 15 }, "grok-2-beta": { input: 2, output: 10 }, "grok-2-mini-beta": { input: 0.5, output: 2.5 }, "grok-2": { input: 2, output: 10 }, "grok-2-mini": { input: 0.5, output: 2.5 }, "grok-beta": { input: 5, output: 15 }, // Default for unspecified xAI models default: { input: 2, output: 10 } } }; var PROVIDER_MODELS = { openai: Object.keys(MODEL_PRICING.openai).filter((key) => key !== "default"), anthropic: Object.keys(MODEL_PRICING.anthropic).filter((key) => key !== "default"), gemini: Object.keys(MODEL_PRICING.gemini).filter((key) => key !== "default"), groq: Object.keys(MODEL_PRICING.groq).filter((key) => key !== "default"), deepseek: Object.keys(MODEL_PRICING.deepseek).filter((key) => key !== "default"), cerebras: Object.keys(MODEL_PRICING.cerebras).filter((key) => key !== "default"), mistral: Object.keys(MODEL_PRICING.mistral).filter((key) => key !== "default"), xai: Object.keys(MODEL_PRICING.xai).filter((key) => key !== "default") }; // src/config.ts var DEFAULT_OPENAI_MODEL = "gpt-4o"; var DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; var config = new Conf({ projectName: "make-abstract", schema: { aiProvider: { type: "string", enum: ["openai", "gemini", "anthropic", "groq", "deepseek", "cerebras", "mistral", "xai"], description: "AI provider to use for generating abstracts" }, openaiApiKey: { type: "string", description: "OpenAI API key" }, geminiApiKey: { type: "string", description: "Google Gemini API key" }, anthropicApiKey: { type: "string", description: "Anthropic API key" }, groqApiKey: { type: "string", description: "Groq API key" }, deepseekApiKey: { type: "string", description: "DeepSeek API key" }, cerebrasApiKey: { type: "string", description: "Cerebras API key" }, mistralApiKey: { type: "string", description: "Mistral API key" }, xaiApiKey: { type: "string", description: "xAI API key" }, zoteroApiKey: { type: "string", description: "Zotero API key" }, zoteroGroupId: { type: "string", description: "Zotero group ID" }, temperature: { type: "number", default: 0.7, minimum: 0, maximum: 1, description: "Sampling temperature for generation" }, openaiModel: { type: "string", default: DEFAULT_OPENAI_MODEL, description: "OpenAI model to use" }, geminiModel: { type: "string", default: DEFAULT_GEMINI_MODEL, description: "Google Gemini model to use" }, anthropicModel: { type: "string", description: "Anthropic model to use" }, groqModel: { type: "string", description: "Groq model to use" }, deepseekModel: { type: "string", description: "DeepSeek model to use" }, cerebrasModel: { type: "string", description: "Cerebras model to use" }, mistralModel: { type: "string", description: "Mistral model to use" }, xaiModel: { type: "string", description: "xAI model to use" }, replacePattern: { type: "string", description: "Regex pattern to match abstracts that should be replaced. If not set, no abstracts will be replaced." } } }); var getConfig = () => config; var setConfig = (key, value) => { config.set(key, value); }; var listConfig = () => { return config.store; }; var hasRequiredConfig = () => { const store = config.store; if (!store.aiProvider) return false; if (store.aiProvider === "openai" && !store.openaiApiKey) return false; if (store.aiProvider === "gemini" && !store.geminiApiKey) return false; if (store.aiProvider === "anthropic" && !store.anthropicApiKey) return false; if (store.aiProvider === "groq" && !store.groqApiKey) return false; if (store.aiProvider === "deepseek" && !store.deepseekApiKey) return false; if (store.aiProvider === "cerebras" && !store.cerebrasApiKey) return false; if (store.aiProvider === "mistral" && !store.mistralApiKey) return false; if (store.aiProvider === "xai" && !store.xaiApiKey) return false; return store.zoteroApiKey && store.zoteroGroupId; }; var hasRequiredAIConfig = () => { const store = config.store; if (!store.aiProvider) return false; if (store.aiProvider === "openai" && !store.openaiApiKey) return false; if (store.aiProvider === "gemini" && !store.geminiApiKey) return false; if (store.aiProvider === "anthropic" && !store.anthropicApiKey) return false; if (store.aiProvider === "groq" && !store.groqApiKey) return false; if (store.aiProvider === "deepseek" && !store.deepseekApiKey) return false; if (store.aiProvider === "cerebras" && !store.cerebrasApiKey) return false; if (store.aiProvider === "mistral" && !store.mistralApiKey) return false; if (store.aiProvider === "xai" && !store.xaiApiKey) return false; return true; }; // src/setup.ts import prompts from "prompts"; async function setupConfig() { const config2 = getConfig(); if (!config2.store.aiProvider) { const { provider } = await prompts({ type: "select", name: "provider", message: "Which AI provider would you like to use?", choices: [ { title: "OpenAI", value: "openai" }, { title: "Gemini", value: "gemini" } ] }); setConfig("aiProvider", provider); } if (config2.store.aiProvider === "openai" && !config2.store.openaiApiKey) { const { apiKey } = await prompts({ type: "text", name: "apiKey", message: "Enter your OpenAI API key" }); setConfig("openaiApiKey", apiKey); } else if (config2.store.aiProvider === "gemini" && !config2.store.geminiApiKey) { const { apiKey } = await prompts({ type: "text", name: "apiKey", message: "Enter your Google Gemini API key" }); setConfig("geminiApiKey", apiKey); } if (!config2.store.zoteroApiKey) { const { apiKey } = await prompts({ type: "text", name: "apiKey", message: "Enter your Zotero API key" }); setConfig("zoteroApiKey", apiKey); } if (!config2.store.zoteroGroupId) { const { groupId } = await prompts({ type: "text", name: "groupId", message: "Enter your Zotero group ID" }); setConfig("zoteroGroupId", groupId); } if (config2.store.aiProvider === "openai") { const { customModel } = await prompts({ type: "confirm", name: "customModel", message: `Would you like to use a different OpenAI model? (default: ${DEFAULT_OPENAI_MODEL})`, initial: false }); if (customModel) { const { model } = await prompts({ type: "text", name: "model", message: "Enter the OpenAI model name", initial: DEFAULT_OPENAI_MODEL }); setConfig("openaiModel", model); } } else if (config2.store.aiProvider === "gemini") { const { customModel } = await prompts({ type: "confirm", name: "customModel", message: `Would you like to use a different Gemini model? (default: ${DEFAULT_GEMINI_MODEL})`, initial: false }); if (customModel) { const { model } = await prompts({ type: "text", name: "model", message: "Enter the Gemini model name", initial: DEFAULT_GEMINI_MODEL }); setConfig("geminiModel", model); } } } // src/abstract.ts import * as path2 from "path"; import * as fs5 from "fs"; // src/zotero.ts import fetch from "node-fetch"; var ZOTERO_API_URL = "https://api.zotero.org"; function buildBasePath(groupId) { const config2 = getConfig(); return groupId ? `/groups/${groupId}` : config2.store.zoteroGroupId ? `/groups/${config2.store.zoteroGroupId}` : `/users/current`; } async function zoteroRequest(path4, options = {}) { const config2 = getConfig(); const response = await fetch(`${ZOTERO_API_URL}${path4}`, { ...options, headers: { "Zotero-API-Version": "3", "Authorization": `Bearer ${config2.store.zoteroApiKey}`, ...options.headers || {} } }); if (!response.ok) { throw new Error(`Zotero API error: ${response.status} ${response.statusText}`); } return response; } async function downloadPDF(url) { console.log(url); const response = await zoteroRequest(url); if (!response.ok) throw new Error(`Failed to download PDF: ${response.statusText}`); return Buffer.from(await response.arrayBuffer()); } async function fetchZoteroItem(itemKey, groupId) { const basePath = buildBasePath(groupId); const itemResponse = await zoteroRequest(`${basePath}/items/${itemKey}`); return await itemResponse.json(); } async function fetchZoteroAttachments(itemKey, groupId) { const basePath = buildBasePath(groupId); const attachmentsResponse = await zoteroRequest(`${basePath}/items/${itemKey}/children`); return await attachmentsResponse.json(); } async function updateZoteroItemAbstract(itemKey, abstract, version, groupId) { const basePath = buildBasePath(groupId); await zoteroRequest(`${basePath}/items/${itemKey}`, { method: "PATCH", headers: { "Content-Type": "application/json", "If-Unmodified-Since-Version": version.toString() }, body: JSON.stringify({ abstractNote: abstract }) }); } async function createZoteroNote(itemKey, noteContent, groupId) { const basePath = buildBasePath(groupId); await zoteroRequest(`${basePath}/items`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify([{ itemType: "note", parentItem: itemKey, note: noteContent }]) }); } function filterPDFAttachments(attachments) { const allPdfAttachments = attachments.filter( (att) => att.data.itemType === "attachment" && att.data.contentType === "application/pdf" ); if (allPdfAttachments.length === 0) { throw new Error("No PDF attachments found"); } if (allPdfAttachments.length === 1) { console.log("Using single PDF attachment (no tag filtering applied)"); return allPdfAttachments; } else { const taggedPdfAttachments = allPdfAttachments.filter( (att) => att.data.tags?.some((tag) => tag.tag === "_publish") ); if (taggedPdfAttachments.length > 0) { console.log(`Using ${taggedPdfAttachments.length} PDF(s) with _publish tag out of ${allPdfAttachments.length} total PDFs`); return taggedPdfAttachments; } else { console.log(`No PDFs with _publish tag found. Using first PDF as fallback (1 of ${allPdfAttachments.length} total PDFs)`); return [allPdfAttachments[0]]; } } } function buildFileDownloadPath(attachmentKey, groupId) { const basePath = buildBasePath(groupId); return `${basePath}/items/${attachmentKey}/file`; } // src/pdf.ts import { definePDFJSModule, extractText, getDocumentProxy } from "unpdf"; import * as fs from "fs"; await definePDFJSModule(() => import("pdfjs-dist/legacy/build/pdf.mjs")); async function extractTextFromPDF(buffer) { const pdf = await getDocumentProxy(new Uint8Array(buffer), { verbosity: 0 }); const { text } = await extractText(pdf, { mergePages: true }); return text; } async function extractTextFromPDFFile(filePath) { const buffer = fs.readFileSync(filePath); return extractTextFromPDF(buffer); } // src/ai.ts import { generateText, generateObject } from "ai"; import { createOpenAI } from "@ai-sdk/openai"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createAnthropic } from "@ai-sdk/anthropic"; import { createGroq } from "@ai-sdk/groq"; import { createDeepSeek } from "@ai-sdk/deepseek"; import { createCerebras } from "@ai-sdk/cerebras"; import { createMistral } from "@ai-sdk/mistral"; import { createXai } from "@ai-sdk/xai"; // src/screening.ts import { z } from "zod"; function parseScreeningQuestions(promptContent) { const lines = promptContent.trim().split("\n").filter((line) => line.trim()); const questions = []; for (const line of lines) { const parts = line.split("|"); if (parts.length !== 3) { throw new Error(`Invalid screening question format: "${line}". Expected format: key|type|question`); } const [key, type, question] = parts.map((part) => part.trim()); if (!/^[a-zA-Z0-9_-]+$/.test(key)) { throw new Error(`Invalid key "${key}": key must contain only letters, numbers, underscores, and dashes`); } if (!["array", "string", "boolean", "number"].includes(type)) { throw new Error(`Invalid type "${type}": type must be one of: array, string, boolean, number`); } questions.push({ key, type, question }); } return questions; } function createZodSchema(questions) { const schemaProperties = {}; for (const question of questions) { switch (question.type) { case "string": schemaProperties[question.key] = z.string().describe(question.question); break; case "boolean": schemaProperties[question.key] = z.boolean().describe(question.question); break; case "array": schemaProperties[question.key] = z.array(z.string()).describe(question.question); break; case "number": schemaProperties[question.key] = z.number().describe(question.question); break; } } return z.object(schemaProperties); } async function updateScreeningTags(itemKey, item, questions, screeningResults, tagPrefix = "_a:", groupId) { const basePath = buildBasePath(groupId); const currentTags = item.data.tags || []; let updatedTags = [...currentTags]; let tagsChanged = false; for (const question of questions) { const questionTagPrefix = `${tagPrefix}${question.key}=`; const existingTagIndex = updatedTags.findIndex( (tagObj) => tagObj.tag.startsWith(questionTagPrefix) ); if (existingTagIndex !== -1) { updatedTags.splice(existingTagIndex, 1); tagsChanged = true; } if (question.type === "boolean") { const newValue = screeningResults[question.key]; const newTag = `${questionTagPrefix}${newValue}`; updatedTags.push({ tag: newTag }); tagsChanged = true; console.log(`Updated tag: ${newTag}`); } else { console.log(`Skipped tag for non-boolean question: ${question.key} (${question.type})`); } } if (tagsChanged) { console.log("Updating item tags..."); const response = await zoteroRequest(`${basePath}/items/${itemKey}`, { method: "PATCH", headers: { "Content-Type": "application/json", "If-Unmodified-Since-Version": item.version.toString() }, body: JSON.stringify({ tags: updatedTags }) }); console.log("Tags updated successfully!"); const newVersion = response.headers.get("Last-Modified-Version"); return newVersion ? parseInt(newVersion) : item.version + 1; } else { console.log("No tag changes needed."); return item.version; } } // src/cost.ts import * as fs2 from "fs"; var CostTracker = class { enabled = false; apiCalls = []; enable() { this.enabled = true; this.apiCalls = []; } isEnabled() { return this.enabled; } trackAPICall(operation, usage, input) { if (!this.enabled) return ""; const config2 = getConfig(); const provider = config2.store.aiProvider ?? "openai"; const model = this.getCurrentModel(); const id = `api_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; const costs = this.calculateCosts(provider, model, usage); const apiCall = { id, timestamp: (/* @__PURE__ */ new Date()).toISOString(), provider, model, operation, input: input ? input.substring(0, 100) + (input.length > 100 ? "..." : "") : void 0, usage, costs }; this.apiCalls.push(apiCall); return id; } calculateCosts(provider, model, usage) { const { inputPricePerMillion, outputPricePerMillion } = this.getModelPricing(provider, model); const inputCost = usage.promptTokens / 1e6 * inputPricePerMillion; const outputCost = usage.completionTokens / 1e6 * outputPricePerMillion; const totalCost = inputCost + outputCost; return { inputCost, outputCost, totalCost }; } getModelPricing(provider, model) { const providerPricing = MODEL_PRICING[provider]; if (!providerPricing) { console.warn(`Unknown provider: ${provider}, using default pricing`); return { inputPricePerMillion: 1, outputPricePerMillion: 3 }; } const modelPricing = providerPricing[model] || providerPricing.default; return { inputPricePerMillion: modelPricing.input, outputPricePerMillion: modelPricing.output }; } getCurrentModel() { const config2 = getConfig(); const provider = config2.store.aiProvider ?? "openai"; switch (provider) { case "openai": return config2.store.openaiModel || "gpt-4o-mini"; case "anthropic": return config2.store.anthropicModel || "claude-3-5-sonnet-20241022"; case "gemini": return config2.store.geminiModel || "gemini-1.5-flash"; case "groq": return config2.store.groqModel || "llama-3.3-70b-versatile"; case "deepseek": return config2.store.deepseekModel || "deepseek-chat"; case "cerebras": return config2.store.cerebrasModel || "llama3.1-70b"; case "mistral": return config2.store.mistralModel || "mistral-large-latest"; case "xai": return config2.store.xaiModel || "grok-2"; default: return "gpt-4o-mini"; } } generateReport() { const config2 = getConfig(); const provider = config2.store.aiProvider ?? "openai"; const totalCosts = this.apiCalls.reduce( (acc, call) => ({ inputCost: acc.inputCost + call.costs.inputCost, outputCost: acc.outputCost + call.costs.outputCost, totalCost: acc.totalCost + call.costs.totalCost, totalTokens: acc.totalTokens + call.usage.totalTokens, totalPromptTokens: acc.totalPromptTokens + call.usage.promptTokens, totalCompletionTokens: acc.totalCompletionTokens + call.usage.completionTokens }), { inputCost: 0, outputCost: 0, totalCost: 0, totalTokens: 0, totalPromptTokens: 0, totalCompletionTokens: 0 } ); return { totalCost: totalCosts, apiCalls: this.apiCalls, metadata: { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), provider, currency: "USD" } }; } saveCostReport(filename) { if (!this.enabled) { console.warn("Cost tracking is not enabled"); return; } const report = this.generateReport(); const filepath = filename.endsWith(".json") ? filename : `${filename}.json`; try { fs2.writeFileSync(filepath, JSON.stringify(report, null, 2)); console.log(` Cost report saved to: ${filepath}`); console.log(`Total cost: $${report.totalCost.totalCost.toFixed(4)}`); console.log(`Total tokens: ${report.totalCost.totalTokens.toLocaleString()}`); console.log(`API calls tracked: ${report.apiCalls.length}`); } catch (error) { console.error(`Failed to save cost report: ${error instanceof Error ? error.message : "unknown error"}`); } } }; var costTracker = new CostTracker(); // src/ai.ts import * as fs3 from "fs"; function createAIModel() { const config2 = getConfig(); const provider = config2.store.aiProvider; switch (provider) { case "openai": { const openai = createOpenAI({ apiKey: config2.store.openaiApiKey }); return openai(config2.store.openaiModel || DEFAULT_OPENAI_MODEL); } case "gemini": { const google = createGoogleGenerativeAI({ apiKey: config2.store.geminiApiKey }); return google(config2.store.geminiModel || DEFAULT_GEMINI_MODEL); } case "anthropic": { const anthropic = createAnthropic({ apiKey: config2.store.anthropicApiKey }); return anthropic(config2.store.anthropicModel || "claude-3-5-sonnet-20241022"); } case "groq": { const groq = createGroq({ apiKey: config2.store.groqApiKey }); return groq(config2.store.groqModel || "llama-3.3-70b-versatile"); } case "deepseek": { const deepseek = createDeepSeek({ apiKey: config2.store.deepseekApiKey }); return deepseek(config2.store.deepseekModel || "deepseek-chat"); } case "cerebras": { const cerebras = createCerebras({ apiKey: config2.store.cerebrasApiKey }); return cerebras(config2.store.cerebrasModel || "llama3.1-70b"); } case "mistral": { const mistral = createMistral({ apiKey: config2.store.mistralApiKey }); return mistral(config2.store.mistralModel || "mistral-large-latest"); } case "xai": { const xai = createXai({ apiKey: config2.store.xaiApiKey }); return xai(config2.store.xaiModel || "grok-3"); } default: throw new Error(`Unsupported AI provider: ${provider}`); } } async function generateAbstractWithAI(text, customPrompt, mode = "text", zoteroId, groupId) { const config2 = getConfig(); const model = createAIModel(); if (mode === "json") { if (!customPrompt) { throw new Error("Custom prompt is required for JSON mode"); } const questions = parseScreeningQuestions(customPrompt); const schema = createZodSchema(questions); const questionsText = questions.map((q) => `${q.key} (${q.type}): ${q.question}`).join("\n"); const systemPrompt = `You are an AI assistant that answers screening questions based on academic documents. IMPORTANT OCR TEXT PROCESSING: The document text may contain OCR errors including merged words, character substitutions, and spelling mistakes. When reading the text, intelligently interpret and correct these errors to understand the intended meaning. Fix merged words, character substitutions like "efective" \u2192 "effective", and obvious spelling mistakes to ensure accurate comprehension of the content. Based on the following document, answer each question accurately. For boolean questions, respond with true or false. For string questions, provide a concise answer. For array questions, provide a list of relevant items. For number questions, provide a numeric value. Questions to answer: ${questionsText} Document content: ${text}`; const { object, usage } = await generateObject({ model, schema, prompt: systemPrompt, temperature: config2.store.temperature }); if (costTracker.isEnabled()) { costTracker.trackAPICall("generateObject", usage, text); } const wrappedResult = { result: object, zotero_id: zoteroId || null, group_id: groupId || config2.store.zoteroGroupId || null, date_generated: (/* @__PURE__ */ new Date()).toISOString(), model: getModelName(config2), prompt: customPrompt }; const result = JSON.stringify(wrappedResult, null, 2); console.log("JSON response:", result); return { result, questions, parsedResults: object }; } else { const defaultPrompt = `Create a concise academic abstract for the following text. The abstract should summarize the main points, methodology, and conclusions. IMPORTANT OCR TEXT PROCESSING: The input text may contain OCR errors including merged words, character substitutions, and spelling mistakes. Please interpret the text intelligently to understand the intended meaning, fixing: - Merged words: "researchersconducted" \u2192 "researchers conducted" - Character substitutions: "efective" \u2192 "effective", "recieve" \u2192 "receive" - Common OCR errors: "rn" mistaken for "m", "cl" for "d", etc. - Spelling mistakes that obscure meaning: "introasting" \u2192 "interesting" - Missing spaces and punctuation where context makes it clear Focus on making the text as legible and academically appropriate as possible while maintaining the original meaning and intent. IMPORTANT: Respond with ONLY the abstract text. Do not include any markdown formatting, headings, annotations, or explanatory text. Provide just the plain abstract content in a single paragraph.`; const prompt = customPrompt || defaultPrompt; const fullPrompt = `${prompt} ${text}`; const { text: response, usage } = await generateText({ model, prompt: fullPrompt, temperature: config2.store.temperature }); if (costTracker.isEnabled()) { costTracker.trackAPICall("generateText", usage, text); } if (!response) { throw new Error(`No response from ${config2.store.aiProvider}`); } console.log("Full response:", response); return { result: response }; } } function getModelName(config2) { const provider = config2.store.aiProvider; switch (provider) { case "openai": return config2.store.openaiModel || DEFAULT_OPENAI_MODEL; case "gemini": return config2.store.geminiModel || DEFAULT_GEMINI_MODEL; case "anthropic": return config2.store.anthropicModel || "claude-3-5-sonnet-20241022"; case "groq": return config2.store.groqModel || "llama-3.3-70b-versatile"; case "deepseek": return config2.store.deepseekModel || "deepseek-chat"; case "cerebras": return config2.store.cerebrasModel || "llama3.1-70b"; case "mistral": return config2.store.mistralModel || "mistral-large-latest"; case "xai": return config2.store.xaiModel || "grok-3"; default: return "unknown"; } } async function generateScanWithAI(pdfPath, customPrompt) { const config2 = getConfig(); const model = createAIModel(); const defaultPrompt = `You are an intelligent document transcription system. Your task is to extract ALL text content from this document in a simple, clean format. TRANSCRIPTION REQUIREMENTS: 1. **Extract ALL text content**: Capture every piece of readable text including: - Headers, titles, and headings - Body text and paragraphs - Captions and labels - Footnotes and annotations - Page numbers and reference numbers - All readable content regardless of formatting 2. **Text quality and accuracy**: - Apply intelligent OCR correction for merged words, character substitutions, spelling errors - Convert handwriting to most likely intended text - Fix obvious recognition mistakes while preserving original meaning - Maintain logical text flow and paragraph structure 3. **For tables and structured data**: - Convert tables to simple key: value format - Extract field labels and their corresponding values as key: value pairs - For multi-column tables, create key: value pairs for each data point - Checkboxes/Multiple choice: Show as key: selected_option - **Selection indicators**: Pay close attention to what is marked as selected: * Crossed-out text/options = SELECTED (this indicates choice, not correction) * Checkmarks (\u2713, \u2717, \u2714) = SELECTED * Circled options = SELECTED * Highlighted or underlined options = SELECTED * X marks on checkboxes = SELECTED - Empty fields: Show as key: (empty) or key: N/A 4. **Selection interpretation rules**: - **For multiple choice questions**: Look for ANY marking that indicates selection: * Crossed-out options (line through text) = SELECTED * Checkmarks or X marks in boxes = SELECTED * Circled options = SELECTED * Underlined or highlighted text = SELECTED - **For yes/no questions**: Identify which option has any marking - **For fill-in-the-blank**: Extract handwritten or typed text - **When multiple marks exist**: Report all marked options, noting if multiple selections appear intended 5. **Output format**: - Present content in logical reading order (top to bottom, left to right) - Use simple key: value format for any structured data - Write paragraphs as plain text blocks - No table formatting, no markdown, no special formatting - Separate different sections with blank lines - For selections, clearly show what was marked: key: SELECTED_OPTION CRITICAL: Extract and present ALL readable text content from the document. Pay special attention to selection marks - crossed-out text usually means SELECTED, not ignored. Use simple key: value pairs for structured data and plain paragraphs for text content. OUTPUT FORMAT: Provide clean, simple text with key: value pairs for tables/forms and plain paragraphs for text content. No formatting, no tables, no markdown. When showing selections, clearly indicate what was marked or chosen.`; const prompt = customPrompt || defaultPrompt; const pdfBuffer = fs3.readFileSync(pdfPath); const { text: response, usage } = await generateText({ model, messages: [ { role: "user", content: [ { type: "text", text: prompt }, { type: "file", data: pdfBuffer, mimeType: "application/pdf", filename: pdfPath.split("/").pop() || "document.pdf" } ] } ], temperature: config2.store.temperature }); if (costTracker.isEnabled()) { costTracker.trackAPICall("scan", usage, prompt); } if (!response) { throw new Error(`No response from ${config2.store.aiProvider}`); } console.log("Scan response:", response); return { result: response }; } // src/output.ts import * as fs4 from "fs"; import * as path from "path"; async function handleOutput(destination, abstract, mode, itemKey, item, outputFilename, isFirstItem = true, customPrompt, promptFileArg, tagPrefix, groupId) { switch (destination) { case "print": handlePrintOutput(abstract, mode, customPrompt, itemKey, item?.data.title); break; case "file": await handleFileOutput(abstract, mode, outputFilename, isFirstItem, customPrompt, itemKey, item?.data.title); break; case "note": if (!itemKey) throw new Error("itemKey required for note destination"); await handleNoteOutput(abstract, mode, itemKey, customPrompt, promptFileArg, tagPrefix, groupId); break; default: throw new Error(`Unknown destination: ${destination}`); } } function handlePrintOutput(abstract, mode, customPrompt, itemKey, title) { console.log("\n" + "=".repeat(60)); console.log(mode === "json" ? "GENERATED JSON RESPONSE" : "GENERATED ABSTRACT"); if (title) { console.log(`Title: ${title}`); } if (itemKey) { console.log(`Item Key: ${itemKey}`); } console.log(`Mode: ${mode}`); if (customPrompt) { console.log("Custom Prompt: Yes"); } console.log("=".repeat(60)); console.log(abstract); if (customPrompt) { console.log("\n" + "-".repeat(40)); console.log(mode === "json" ? "SCREENING QUESTIONS USED:" : "CUSTOM PROMPT USED:"); console.log("-".repeat(40)); console.log(customPrompt); } console.log("=".repeat(60) + "\n"); } async function handleFileOutput(abstract, mode, outputFilename, isFirstItem = true, customPrompt, itemKey, title, pdfPath) { let filename; if (outputFilename) { if (mode === "json") { filename = outputFilename.endsWith(".json") ? outputFilename : `${outputFilename}.json`; } else { filename = outputFilename.endsWith(".txt") ? outputFilename : `${outputFilename}.txt`; } } else { const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").split("T")[0]; const baseName = pdfPath ? path.basename(pdfPath, ".pdf") : itemKey; if (mode === "json") { filename = `screening_${baseName}_${timestamp}.json`; } else if (mode === "scan") { filename = `scan_${baseName}_${timestamp}.txt`; } else { filename = `abstract_${baseName}_${timestamp}.txt`; } } const filepath = path.resolve(filename); if (mode === "json") { const jsonData = JSON.parse(abstract); let jsonArray = []; if (!isFirstItem && fs4.existsSync(filepath)) { try { const existingContent = fs4.readFileSync(filepath, "utf8"); jsonArray = JSON.parse(existingContent); if (!Array.isArray(jsonArray)) { jsonArray = [jsonArray]; } } catch (error) { jsonArray = []; } } jsonArray.push(jsonData); fs4.writeFileSync(filepath, JSON.stringify(jsonArray, null, 2), "utf8"); } else { let fileContent = ""; if (!isFirstItem) { fileContent += "\n\n" + "=".repeat(50) + "\n\n"; } if (title) { fileContent += `Title: ${title} `; } else if (pdfPath) { fileContent += `PDF File: ${path.basename(pdfPath)} `; } if (itemKey) { fileContent += `Item Key: ${itemKey} `; } fileContent += `Generated: ${(/* @__PURE__ */ new Date()).toISOString()} `; fileContent += `Mode: ${mode} `; if (customPrompt) { fileContent += `Custom Prompt: Yes `; } fileContent += "=".repeat(50) + "\n\n"; fileContent += abstract; if (customPrompt) { fileContent += "\n\n" + "-".repeat(40) + "\n"; if (mode === "scan") { fileContent += "SCAN PROMPT USED:\n"; } else { fileContent += "CUSTOM PROMPT USED:\n"; } fileContent += "-".repeat(40) + "\n"; fileContent += customPrompt; } if (isFirstItem) { fs4.writeFileSync(filepath, fileContent, "utf8"); } else { fs4.appendFileSync(filepath, fileContent, "utf8"); } } if (mode === "json") { console.log(`Screening completed and saved to: ${filepath}`); } else if (mode === "scan") { console.log(`PDF scan completed and saved to: ${filepath}`); } else { console.log(`Abstract generated and saved to: ${filepath}`); } } async function handleNoteOutput(abstract, mode, itemKey, customPrompt, promptFileArg, tagPrefix, groupId) { const now = /* @__PURE__ */ new Date(); const timestamp = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}, ${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}:${String(now.getSeconds()).padStart(2, "0")}`; const promptFile = promptFileArg ? path.basename(promptFileArg) : "default"; let noteContent; if (mode === "json") { let formattedJson; try { const jsonObj = JSON.parse(abstract); formattedJson = JSON.stringify(jsonObj, null, 2); } catch (e) { formattedJson = abstract; } noteContent = `<p><strong>Screening Results - ${tagPrefix} - ${promptFile} - ${timestamp}.</strong></p><pre>${formattedJson.replace(/\n/g, "<br/>")}</pre>`; } else { noteContent = `<p><strong>Generated Abstract - ${timestamp}.</strong></p><p>${abstract.replace(/\n/g, "</p><p>")}</p>`; } if (customPrompt) { noteContent += mode === "json" ? `<hr><p><strong>Screening Questions Used:</strong></p><p><em>${customPrompt.replace(/\n/g, "</p><p><em>")}</em></p>` : `<hr><p><strong>Custom Prompt Used:</strong></p><p><em>${customPrompt.replace(/\n/g, "</p><p><em>")}</em></p>`; } await createZoteroNote(itemKey, noteContent, groupId); console.log(mode === "json" ? "Screening completed and saved as note attachment!" : "Abstract generated and saved as note attachment!"); } function handleFilePrintOutput(abstract, mode, filePath, customPrompt) { const fileExtension = path.extname(filePath).toLowerCase(); const fileType = fileExtension === ".pdf" ? "PDF" : fileExtension === ".txt" ? "TXT" : "File"; console.log("\n" + "=".repeat(60)); if (mode === "json") { console.log("GENERATED JSON RESPONSE"); } else if (mode === "scan") { console.log("SCANNED CONTENT"); } else { console.log("GENERATED ABSTRACT"); } console.log(`${fileType} File: ${path.basename(filePath)}`); console.log(`Mode: ${mode}`); if (customPrompt) { console.log("Custom Prompt: Yes"); } console.log("=".repeat(60)); console.log(abstract); if (customPrompt) { console.log("\n" + "-".repeat(40)); if (mode === "json") { console.log("SCREENING QUESTIONS USED:"); } else if (mode === "scan") { console.log("SCAN PROMPT USED:"); } else { console.log("CUSTOM PROMPT USED:"); } console.log("-".repeat(40)); console.log(customPrompt); } console.log("=".repeat(60) + "\n"); } async function handleGenericFileOutput(abstract, mode, filePath, outputFilename, isFirstItem = true, customPrompt) { await handleFileOutput(abstract, mode, outputFilename, isFirstItem, customPrompt, void 0, void 0, filePath); } function saveJsonResult(abstract, tagPrefix, itemKey, promptFileArg, item, savePrefix = "result-") { const jsonFilename = `${savePrefix}${tagPrefix}-${itemKey}.json`; const jsonFilepath = path.resolve(jsonFilename); let aiResult; try { aiResult = JSON.parse(abstract); } catch (e) { aiResult = abstract; } const output = { tag_prefix: tagPrefix, prompt_file: promptFileArg ? path.basename(promptFileArg) : "default", zotero_item: item, result: aiResult }; fs4.writeFileSync(jsonFilepath, JSON.stringify(output, null, 2), "utf8"); console.log(`Saved JSON result to ${jsonFilepath}`); } // src/abstract.ts async function generateAbstractFromFile(filePath, destination = "print", customPrompt, outputFilename, mode = "text", isFirstItem = true, tagPrefix = "_a:", saveJson, promptFileArg, saveScan, savePrefix = "result-") { const fileExtension = path2.extname(filePath).toLowerCase(); const isPDF = fileExtension === ".pdf"; const isTXT = fileExtension === ".txt"; if (destination === "abstract" || destination === "note") { throw new Error(`Destination "${destination}" is not supported when processing ${isPDF ? "PDF" : "TXT"} files directly. Use "print" or "file" instead.`); } if (mode === "scan" && !isPDF) { throw new Error("Scan mode is only supported for PDF files"); } let abstract; if (mode === "scan" && isPDF) { console.log("Scanning PDF with AI vision..."); const { result } = await generateScanWithAI(filePath, customPrompt); abstract = result; if (saveScan) { const originalFileName = path2.basename(filePath, path2.extname(filePath)); const scanFileName = `${savePrefix}${originalFileName}.txt`; const scanFilePath = path2.resolve(scanFileName); fs5.writeFileSync(scanFilePath, abstract, "utf8"); console.log(`Scan result saved to: ${scanFileName}`); } } else { let text; if (isPDF) { console.log("Extracting text from PDF..."); text = await extractTextFromPDFFile(filePath); } else if (isTXT) { console.log("Reading text from TXT file..."); text = fs5.readFileSync(filePath, "utf8"); } else { throw new Error(`Unsupported file type: ${fileExtension}`); } if (!text.trim()) { throw new Error(`No text content found in ${isPDF ? "PDF" : "TXT"} file`); } console.log(mode === "json" ? "Generating screening..." : "Generating abstract..."); const { result } = await generateAbstractWithAI(text, customPrompt, mode); abstract = result; } switch (destination) { case "print": handleFilePrintOutput(abstract, mode, filePath, customPrompt); break; case "file": await handleGenericFileOutput(abstract, mode, filePath, outputFilename, isFirstItem, customPrompt); break; default: throw new Error(`Unknown destination: ${destination}`); } } async function generateAbstract(itemKey