pdf-toolmd
Version:
Convert PDF to Markdown using Node.js
541 lines (531 loc) • 27.8 kB
JavaScript
"use strict";
var __webpack_require__ = {};
(()=>{
__webpack_require__.n = (module)=>{
var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
__webpack_require__.d(getter, {
a: getter
});
return getter;
};
})();
(()=>{
__webpack_require__.d = (exports1, definition)=>{
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
enumerable: true,
get: definition[key]
});
};
})();
(()=>{
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
})();
(()=>{
__webpack_require__.r = (exports1)=>{
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
value: 'Module'
});
Object.defineProperty(exports1, '__esModule', {
value: true
});
};
})();
var __webpack_exports__ = {};
__webpack_require__.r(__webpack_exports__);
__webpack_require__.d(__webpack_exports__, {
getPageNum: ()=>getPageNum,
parsePdf: ()=>parsePdf
});
const external_fs_extra_namespaceObject = require("fs-extra");
var external_fs_extra_default = /*#__PURE__*/ __webpack_require__.n(external_fs_extra_namespaceObject);
const external_path_namespaceObject = require("path");
var external_path_default = /*#__PURE__*/ __webpack_require__.n(external_path_namespaceObject);
const pdfium_namespaceObject = require("@hyzyla/pdfium");
const external_sharp_namespaceObject = require("sharp");
var external_sharp_default = /*#__PURE__*/ __webpack_require__.n(external_sharp_namespaceObject);
async function renderFunction(options) {
return await external_sharp_default()(options.data, {
raw: {
width: options.width,
height: options.height,
channels: 4
}
}).png().toBuffer();
}
const getPageCount = async (pdfData)=>{
let data;
data = 'string' == typeof pdfData ? new Uint8Array(await external_fs_extra_default().readFile(pdfData)) : new Uint8Array(pdfData);
const library = await pdfium_namespaceObject.PDFiumLibrary.init();
const document = await library.loadDocument(data);
return document.getPageCount();
};
const generateFullPageImages = async (pdfData, outputDir, scale = 3)=>{
await external_fs_extra_default().ensureDir(outputDir);
let data;
data = 'string' == typeof pdfData ? new Uint8Array(await external_fs_extra_default().readFile(pdfData)) : new Uint8Array(pdfData);
const library = await pdfium_namespaceObject.PDFiumLibrary.init();
const document = await library.loadDocument(data);
const numPages = document.getPageCount();
console.log(`PDF\u{6587}\u{6863}\u{5171} ${numPages} \u{9875}`);
const pageImages = [];
for (const page of document.pages()){
const pageIndex = page.number + 1;
console.log(`\u{5904}\u{7406}\u{7B2C} ${pageIndex} \u{9875}...`);
const image = await page.render({
scale: 3,
render: renderFunction
});
pageImages.push({
index: pageIndex,
data: Buffer.from(image.data)
});
}
document.destroy();
library.destroy();
return pageImages;
};
const extractMdFromLLMOutput = (output)=>{
const mdStart = output.indexOf('```markdown');
const mdEnd = output.lastIndexOf('```');
if (-1 === mdStart || -1 === mdEnd) return void console.error("\u6A21\u578B\u672A\u6309\u6807\u51C6\u683C\u5F0F\u8F93\u51FA:", output);
{
const mdString = output.substring(mdStart + 12, mdEnd);
return mdString;
}
};
const getOldMarkdownHeadings = (markdownText)=>{
const title = [];
const lines = markdownText.split('\n');
lines.forEach((line)=>{
const match = line.match(/^#+\s*(.*)/);
if (match) title.push(line);
});
return title.join('\n');
};
const adjustMarkdownHeadings = (markdownText, newTitle)=>{
const map = createTitleLevelMap(newTitle);
const lines = markdownText.split('\n');
const processedLines = [];
lines.forEach((line)=>{
const match = line.match(/^#+\s*(.*)/);
if (!match) return void processedLines.push(line);
const content = match ? match[1] : line;
let newLine = line;
if (map.has(content)) {
const level = map.get(content);
const hashes = '#'.repeat(level);
newLine = `${hashes} ${content}`;
console.log("\u8F6C\u6362\u524D\uFF1A" + line + "===>\u8F6C\u6362\u540E" + newLine);
}
processedLines.push(newLine);
});
return processedLines.join('\n');
};
function createTitleLevelMap(data, map = new Map()) {
const lines = data.split('\n');
for (const line of lines){
const headerMatch = line.match(/^(#+)\s*(.+)/);
if (headerMatch) {
const level = headerMatch[1].length;
const text = headerMatch[2].trim();
map.set(text, level);
}
}
return map;
}
const external_https_namespaceObject = require("https");
var external_https_default = /*#__PURE__*/ __webpack_require__.n(external_https_namespaceObject);
const external_http_namespaceObject = require("http");
var external_http_default = /*#__PURE__*/ __webpack_require__.n(external_http_namespaceObject);
const DEFAULT_ROLE_PROMPT = `\u{4F60}\u{662F}\u{4E00}\u{4E2A}PDF\u{6587}\u{6863}\u{89E3}\u{6790}\u{5668}\u{FF0C}\u{4F7F}\u{7528}markdown\u{548C}latex\u{8BED}\u{6CD5}\u{8F93}\u{51FA}\u{56FE}\u{7247}\u{7684}\u{5185}\u{5BB9}\u{3002}`;
class ModelClient {
config;
apiKey;
constructor(config){
this.config = {
model: 'gpt-4-vision-preview',
modelConfig: {},
...config
};
this.initClient();
}
initClient() {
const { apiKey } = this.config;
if (!apiKey) throw new Error("\u5FC5\u987B\u63D0\u4F9BAPI\u5BC6\u94A5");
this.apiKey = apiKey;
}
async processImage(imagePath, prompt, options = {}) {
const model = options.model || this.config.model;
if (!model) throw new Error("\u5FC5\u987B\u6307\u5B9A\u6A21\u578B");
const rolePrompt = options.rolePrompt || DEFAULT_ROLE_PROMPT;
const maxTokens = options.maxTokens || 4096;
const endpoint = options.endpoint || this.config.baseUrl;
let base64Image = null;
if (imagePath) base64Image = imagePath.toString('base64');
if (model.startsWith('gpt-4') || model.startsWith('gpt-3.5')) return this.callOpenAIAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint);
if (model.startsWith('claude') && !this.config.openAiApicompatible) return this.callClaudeAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint);
if (model.startsWith('gemini') && !this.config.openAiApicompatible) return this.callGeminiAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint);
if (model.startsWith('doubao') && !this.config.openAiApicompatible) return this.callDoubaoAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint);
return this.callOpenAIAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint);
}
async callOpenAIAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint) {
let apiEndpoint = endpoint || 'https://api.openai.com/v1/chat/completions';
if (!apiEndpoint.includes('/chat/completions')) apiEndpoint = apiEndpoint.endsWith('/') ? `${apiEndpoint}chat/completions` : `${apiEndpoint}/chat/completions`;
const userContent = [
{
type: 'text',
text: prompt
}
];
if (base64Image) userContent.push({
type: 'image_url',
image_url: {
url: `data:image/png;base64,${base64Image}`
}
});
const requestData = {
model,
messages: [
{
role: 'system',
content: rolePrompt
},
{
role: 'user',
content: userContent
}
],
max_tokens: maxTokens
};
const response = await this.makeHttpRequest(apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.apiKey}`
},
body: JSON.stringify(requestData)
});
if (response.error) throw new Error(`OpenAI API\u{8C03}\u{7528}\u{5931}\u{8D25}: ${response.error.message || JSON.stringify(response.error)}`);
return response.choices?.[0]?.message?.content || '';
}
async callClaudeAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint) {
const apiEndpoint = endpoint || 'https://api.anthropic.com/v1/messages';
const messageContent = [
{
type: 'text',
text: prompt
}
];
if (base64Image) messageContent.push({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: base64Image
}
});
const requestData = {
model,
system: rolePrompt,
max_tokens: maxTokens,
messages: [
{
role: 'user',
content: messageContent
}
]
};
const response = await this.makeHttpRequest(apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01'
},
body: JSON.stringify(requestData)
});
if (response.error) throw new Error(`Claude API\u{8C03}\u{7528}\u{5931}\u{8D25}: ${response.error.message || JSON.stringify(response.error)}`);
return response.content?.[0]?.text || '';
}
async callGeminiAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint) {
const apiEndpoint = endpoint || `https://generativelanguage.googleapis.com/v1/models/${model}:generateContent`;
const parts = [
{
text: `${rolePrompt}\n${prompt}`
}
];
if (base64Image) parts.push({
inline_data: {
mime_type: 'image/png',
data: base64Image
}
});
const requestData = {
contents: [
{
parts: parts
}
],
generation_config: {
max_output_tokens: maxTokens
}
};
const fullUrl = `${apiEndpoint}?key=${this.apiKey}`;
const response = await this.makeHttpRequest(fullUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(requestData)
});
if (response.error) throw new Error(`Gemini API\u{8C03}\u{7528}\u{5931}\u{8D25}: ${response.error.message || JSON.stringify(response.error)}`);
return response.candidates?.[0]?.content?.parts?.[0]?.text || '';
}
async callDoubaoAPI(model, rolePrompt, prompt, base64Image, maxTokens, endpoint) {
let apiEndpoint = endpoint || 'https://ark.cn-beijing.volces.com/api/v3';
if (!apiEndpoint.includes('/chat/completions')) apiEndpoint = apiEndpoint.endsWith('/') ? `${apiEndpoint}chat/completions` : `${apiEndpoint}/chat/completions`;
const userContent = [
{
type: 'text',
text: prompt
}
];
if (base64Image) userContent.push({
type: 'image_url',
image_url: {
url: `data:image/png;base64,${base64Image}`
}
});
const requestData = {
model,
messages: [
{
role: 'system',
content: rolePrompt
},
{
role: 'user',
content: userContent
}
],
max_tokens: maxTokens
};
const response = await this.makeHttpRequest(apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(JSON.stringify(requestData)).toString(),
Authorization: `Bearer ${this.apiKey}`
},
body: JSON.stringify(requestData)
});
if (response.error) throw new Error(`\u{8C46}\u{5305}API\u{8C03}\u{7528}\u{5931}\u{8D25}: ${response.error.message || JSON.stringify(response.error)}`);
return response.choices?.[0]?.message?.content || '';
}
async makeHttpRequest(url, options) {
return new Promise((resolve, reject)=>{
const urlObj = new URL(url);
const client = 'https:' === urlObj.protocol ? external_https_default() : external_http_default();
const requestOptions = {
method: options.method || 'GET',
headers: options.headers || {},
hostname: urlObj.hostname,
port: urlObj.port || ('https:' === urlObj.protocol ? 443 : 80),
path: urlObj.pathname + urlObj.search
};
const req = client.request(requestOptions, (res)=>{
let data = '';
res.on('data', (chunk)=>{
data += chunk;
});
res.on('end', ()=>{
try {
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
const parsedData = JSON.parse(data);
resolve(parsedData);
} else reject(new Error(`\u{8BF7}\u{6C42}\u{5931}\u{8D25}\u{FF0C}\u{72B6}\u{6001}\u{7801}: ${res.statusCode}, \u{54CD}\u{5E94}: ${data}`));
} catch (e) {
reject(new Error(`API\u{54CD}\u{5E94}\u{89E3}\u{6790}\u{5931}\u{8D25}: ${e.message}, \u{539F}\u{59CB}\u{54CD}\u{5E94}: ${data}`));
}
});
});
req.on('error', (error)=>{
reject(new Error(`\u{8BF7}\u{6C42}\u{5931}\u{8D25}: ${error.message}`));
});
if (options.body) req.write(options.body);
req.end();
});
}
getSupportedModels() {
return [
'gpt-4-vision-preview',
'gpt-4o',
'claude-3-opus-20240229',
'claude-3-sonnet-20240229',
'gemini-pro-vision',
'doubao-1.5-vision-pro-32k-250115'
];
}
}
const src_modelClient = ModelClient;
const DefaultPrompt = `\u{4F7F}\u{7528}markdown\u{8BED}\u{6CD5}\u{FF0C}\u{5C06}\u{56FE}\u{7247}\u{4E2D}\u{8BC6}\u{522B}\u{5230}\u{7684}\u{6587}\u{5B57}\u{8F6C}\u{6362}\u{4E3A}markdown\u{683C}\u{5F0F}\u{8F93}\u{51FA}\u{3002}\u{4F60}\u{5FC5}\u{987B}\u{505A}\u{5230}\u{FF1A}
1. \u{8F93}\u{51FA}\u{548C}\u{4F7F}\u{7528}\u{8BC6}\u{522B}\u{5230}\u{7684}\u{56FE}\u{7247}\u{7684}\u{76F8}\u{540C}\u{7684}\u{8BED}\u{8A00}\u{FF0C}\u{4F8B}\u{5982}\u{FF0C}\u{8BC6}\u{522B}\u{5230}\u{82F1}\u{8BED}\u{7684}\u{5B57}\u{6BB5}\u{FF0C}\u{8F93}\u{51FA}\u{7684}\u{5185}\u{5BB9}\u{5FC5}\u{987B}\u{662F}\u{82F1}\u{8BED}\u{3002}
2. \u{4E0D}\u{8981}\u{89E3}\u{91CA}\u{548C}\u{8F93}\u{51FA}\u{65E0}\u{5173}\u{7684}\u{6587}\u{5B57}\u{FF0C}\u{76F4}\u{63A5}\u{8F93}\u{51FA}\u{56FE}\u{7247}\u{4E2D}\u{7684}\u{5185}\u{5BB9}\u{3002}
3. \u{5185}\u{5BB9}\u{4E0D}\u{8981}\u{5305}\u{542B}\u{5728}\`\`\`markdown \`\`\`\u{4E2D}\u{3001}\u{6BB5}\u{843D}\u{516C}\u{5F0F}\u{4F7F}\u{7528} $$ $$ \u{7684}\u{5F62}\u{5F0F}\u{3001}\u{884C}\u{5185}\u{516C}\u{5F0F}\u{4F7F}\u{7528} $ $ \u{7684}\u{5F62}\u{5F0F}\u{3002}
4. \u{5FFD}\u{7565}\u{6389}\u{9875}\u{7709}\u{9875}\u{811A}\u{91CC}\u{7684}\u{5185}\u{5BB9}
5. \u{8BF7}\u{4E0D}\u{8981}\u{5BF9}\u{56FE}\u{7247}\u{7684}\u{6807}\u{9898}\u{8FDB}\u{884C}markdown\u{7684}\u{683C}\u{5F0F}\u{5316}\u{FF0C}\u{76F4}\u{63A5}\u{4EE5}\u{6587}\u{672C}\u{5F62}\u{5F0F}\u{8F93}\u{51FA}\u{5230}\u{5185}\u{5BB9}\u{4E2D}\u{3002}
6. \u{6709}\u{53EF}\u{80FD}\u{6BCF}\u{9875}\u{90FD}\u{4F1A}\u{51FA}\u{73B0}\u{671F}\u{520A}\u{540D}\u{79F0}\u{FF0C}\u{8BBA}\u{6587}\u{540D}\u{79F0}\u{FF0C}\u{4F1A}\u{8BAE}\u{540D}\u{79F0}\u{6216}\u{8005}\u{4E66}\u{7C4D}\u{540D}\u{79F0}\u{FF0C}\u{8BF7}\u{5FFD}\u{7565}\u{4ED6}\u{4EEC}\u{4E0D}\u{8981}\u{8BC6}\u{522B}\u{6210}\u{6807}\u{9898}
7. \u{8BF7}\u{7CBE}\u{786E}\u{5206}\u{6790}\u{5F53}\u{524D}PDF\u{9875}\u{9762}\u{7684}\u{6587}\u{672C}\u{7ED3}\u{6784}\u{548C}\u{89C6}\u{89C9}\u{5E03}\u{5C40}\u{FF0C}\u{6309}\u{4EE5}\u{4E0B}\u{8981}\u{6C42}\u{5904}\u{7406}\u{FF1A}
1. \u{8BC6}\u{522B}\u{6240}\u{6709}\u{6807}\u{9898}\u{6587}\u{672C}\u{FF0C}\u{5E76}\u{5224}\u{65AD}\u{5176}\u{5C42}\u{7EA7}\u{FF08}\u{6839}\u{636E}\u{5B57}\u{4F53}\u{5927}\u{5C0F}\u{3001}\u{52A0}\u{7C97}\u{3001}\u{4F4D}\u{7F6E}\u{7B49}\u{89C6}\u{89C9}\u{7279}\u{5F81}\u{FF09}
2. \u{8F93}\u{51FA}\u{4E3A}\u{5E26}\u{5C42}\u{7EA7}\u{7684}Markdown\u{683C}\u{5F0F}\u{FF0C}\u{4E25}\u{683C}\u{4F7F}\u{7528}\u{4EE5}\u{4E0B}\u{89C4}\u{5219}\u{FF1A}
- \u{4E00}\u{7EA7}\u{6807}\u{9898}\u{FF1A}\u{5B57}\u{4F53}\u{6700}\u{5927}/\u{9876}\u{90E8}\u{5C45}\u{4E2D}\u{FF0C}\u{524D}\u{9762}\u{52A0} #
- \u{4E8C}\u{7EA7}\u{6807}\u{9898}\u{FF1A}\u{5B57}\u{4F53}\u{8F83}\u{5927}/\u{5DE6}\u{5BF9}\u{9F50}\u{52A0}\u{7C97}\u{FF0C}\u{6709}\u{53EF}\u{80FD}\u{662F}\u{6570}\u{5B57}\u{5F00}\u{5934}\u{4E5F}\u{6709}\u{53EF}\u{80FD}\u{662F}\u{7F57}\u{9A6C}\u{6570}\u{7EC4}\u{5F00}\u{5934}\u{FF0C}\u{524D}\u{9762}\u{52A0} ##
- \u{4E09}\u{7EA7}\u{6807}\u{9898}\u{FF1A}\u{5B57}\u{4F53}\u{7A0D}\u{5927}/\u{5DE6}\u{5BF9}\u{9F50}\u{52A0}\u{7C97}\u{FF0C}\u{524D}\u{9762}\u{52A0} ###
- \u{6B63}\u{6587}\u{6587}\u{672C}\u{FF1A}\u{76F4}\u{63A5}\u{8F6C}\u{6362}\u{4E3A}\u{666E}\u{901A}\u{6BB5}\u{843D}
3. \u{4E0D}\u{786E}\u{5B9A}\u{5C42}\u{7EA7}\u{7684}\u{6807}\u{9898}\u{8BF7}\u{6807}\u{8BB0}[?]
4. \u{5982}\u{679C}\u{662F}\u{4E2D}\u{6587}\u{6587}\u{732E}\u{FF0C}\u{4F46}\u{662F}\u{6709}\u{82F1}\u{6587}\u{6807}\u{9898}\u{548C}\u{6458}\u{8981}\u{53EF}\u{4EE5}\u{7701}\u{7565}\u{4E0D}\u{8F93}\u{51FA}
\u{793A}\u{4F8B}\u{8F93}\u{51FA}\u{FF1A}
## 4\u{7814}\u{7A76}\u{65B9}\u{6CD5}
### 4.1\u{6570}\u{636E}\u{6536}\u{96C6}
\u{672C}\u{6587}\u{91C7}\u{7528}\u{95EE}\u{5377}\u{8C03}\u{67E5}...
`;
const DefaultTextPrompt = `\u{4F60}\u{662F}\u{4E00}\u{4E2A}\u{4E13}\u{4E1A}\u{7684}\u{6587}\u{672C}\u{7ED3}\u{6784}\u{5316}\u{5904}\u{7406}\u{52A9}\u{624B}\u{FF0C}\u{64C5}\u{957F}\u{6839}\u{636E}\u{524D}\u{7F00}\u{89C4}\u{5219}\u{548C}\u{6807}\u{9898}\u{8BED}\u{4E49}\u{5206}\u{6790}\u{5E76}\u{4F18}\u{5316}Markdown\u{6587}\u{6863}\u{7684}\u{6807}\u{9898}\u{5C42}\u{7EA7}\u{7ED3}\u{6784}\u{3002}\u{8BF7}\u{6839}\u{636E}\u{4EE5}\u{4E0B}\u{8981}\u{6C42}\u{5904}\u{7406}\u{6211}\u{63D0}\u{4F9B}\u{7684}Markdown\u{6807}\u{9898}\u{FF1A}
## \u{4EFB}\u{52A1}\u{63CF}\u{8FF0}
\u{8BF7}\u{6839}\u{636E}markdown\u{6587}\u{7AE0}\u{6807}\u{9898}\u{7684}\u{5B9E}\u{9645}\u{542B}\u{4E49}\u{FF0C}\u{4EE5}\u{53CA}\u{6807}\u{9898}\u{7684}\u{524D}\u{7F00}\u{7279}\u{5F81}\u{8C03}\u{6574}\u{5404}\u{7EA7}\u{6807}\u{9898}\u{7684}\u{6B63}\u{786E}\u{5C42}\u{7EA7}\u{5173}\u{7CFB}\u{FF0C}\u{5177}\u{4F53}\u{8981}\u{6C42}\u{5982}\u{4E0B}\u{FF1A}
1. \u{4E00}\u{822C}\u{76F8}\u{540C}\u{683C}\u{5F0F}\u{7684}\u{524D}\u{7F00}\u{7684}\u{6807}\u{9898}\u{662F}\u{540C}\u{7EA7}\u{5173}\u{7CFB}({title}\u{4EE3}\u{8868}\u{5B9E}\u{9645}\u{7684}\u{6807}\u{9898}\u{5185}\u{5BB9})\u{FF1A}
\u{4F8B}\u{5982}\u{FF1A}
\u{7EAF}\u{6570}\u{5B57}\u{524D}\u{7F00}\u{5F00}\u{5934}\`1 {title}\`\u{FF0C} \` 2 {title}\` \u{FF0C}\` 3 {title}\`\u{FF0C}\` 4 {title}\`\u{FF0C}\` 5 {title}\` ... \u{7B49}
\u{7F57}\u{9A6C}\u{6570}\u{5B57}\u{524D}\u{7F00}\u{5F00}\u{5934}\u{7684}\`I {title}\`\u{FF0C}\`II {title}\` \u{FF0C}\`III {title}\`\u{FF0C}\`IV {title}\`\u{FF0C}\`V {title}\` ... \u{7B49}
\u{5C0F}\u{6570}\u{70B9}\u{5206}\u{9694}\u{6570}\u{7EC4}\u{524D}\u{7F00}\u{5F00}\u{5934} \`1.1 {title}\`, \`1.2 {title}\`, \`1.3 {title}\`.... \`2.1 {title}\`, \`2.2 {title}\` \u{7B49}
2. \u{5C06}\u{5B50}\u{6807}\u{9898}\u{6B63}\u{786E}\u{5D4C}\u{5957}\u{5230}\u{7236}\u{6807}\u{9898}\u{4E0B}\u{FF08}\u{5982}\`1.1 {title}\`\u{5E94}\u{4F5C}\u{4E3A}\`1 {title}\`\u{7684}\u{5B50}\u{6807}\u{9898}\u{FF09}
3. \u{5254}\u{9664}\u{4E0E}\u{6587}\u{7AE0}\u{5185}\u{5BB9}\u{65E0}\u{5173}\u{7684}\u{6807}\u{9898}
4. \u{4FDD}\u{6301}\u{8F93}\u{51FA}\u{6807}\u{9898}\u{5185}\u{5BB9}\u{4E0E}\u{8F93}\u{5165}\u{5B8C}\u{5168}\u{4E00}\u{81F4}
5. \u{786E}\u{4FDD}\u{5185}\u{5BB9}\u{65E0}\u{7F3A}\u{5931}
6. \u{5982}\u{679C}\u{662F}\u{4E2D}\u{6587}\u{6587}\u{732E}\u{FF0C}\u{4F46}\u{6709}\u{82F1}\u{6587}\u{7684}\u{6587}\u{7AE0}\u{9898}\u{76EE}\u{FF0C}\u{53EF}\u{4EE5}\u{7701}\u{7565}
## \u{8F93}\u{5165}\u{8F93}\u{51FA}\u{683C}\u{5F0F}
- \u{8F93}\u{5165}\u{FF1A}\u{5305}\u{542B}\u{9519}\u{8BEF}\u{5C42}\u{7EA7}\u{5173}\u{7CFB}\u{7684}markdown\u{6807}\u{9898}\u{7ED3}\u{6784}
- \u{8F93}\u{51FA}\u{FF1A}\u{4FEE}\u{6B63}\u{540E}\u{7684}\u{6807}\u{51C6}markdown\u{6807}\u{9898}\u{5C42}\u{7EA7}\u{7ED3}\u{6784}
## \u{5904}\u{7406}\u{539F}\u{5219}
1. \u{4E25}\u{683C}\u{6839}\u{636E}\u{6807}\u{9898}\u{8BED}\u{4E49}\u{786E}\u{5B9A}\u{6240}\u{5C5E}\u{5173}\u{7CFB}
2. \u{4EC5}\u{8C03}\u{6574}\u{5C42}\u{7EA7}\u{4E0D}\u{4FEE}\u{6539}\u{539F}\u{6807}\u{9898}\u{6587}\u{672C}
3. \u{65E0}\u{5173}\u{6807}\u{9898}\u{76F4}\u{63A5}\u{79FB}\u{9664}\u{4E0D}\u{4FDD}\u{7559}\u{5360}\u{4F4D}
4. \u{76F8}\u{540C}\u{524D}\u{7F00}\u{89C4}\u{5219}\u{7684}\u{6807}\u{9898}\u{5FC5}\u{987B}\u{662F}\u{540C}\u{4E00}\u{7EA7}\u{522B}\u{FF0C}\u{4E0D}\u{80FD}\u{51FA}\u{73B0} \u{4E00}\u{90E8}\u{5206}\u{662F} n\u{7EA7}\u{6807}\u{9898}\u{FF0C}\u{4E00}\u{90E8}\u{5206}\u{662F}\u{5176}\u{4ED6}\u{7EA7}\u{522B}\u{7684}\u{6807}\u{9898}
## \u{8F93}\u{51FA}\u{8981}\u{6C42}
\u{8BF7}\u{5C06}\u{4FEE}\u{6B63}\u{540E}\u{7684}\u{5B8C}\u{6574}\u{6807}\u{9898}\u{7ED3}\u{6784}\u{653E}\u{5728}\u{4EE3}\u{7801}\u{5757}\u{4E2D}\u{8FD4}\u{56DE}\u{FF0C}\u{683C}\u{5F0F}\u{793A}\u{4F8B}\u{5982}\u{4E0B}\u{FF1A}
\u{671F}\u{671B}\u{8F93}\u{51FA}\u{FF1A}
\`\`\`markdown
\`\`\`
\u{8BF7}\u{5904}\u{7406}\u{4EE5}\u{4E0B}\u{6570}\u{636E}\u{FF1A}
`;
const parsePdf = async (pdfPath, options = {})=>{
const { outputDir = './output', apiKey = "", baseUrl, openAiApicompatible = false, model = "", prompt = DefaultPrompt, textPrompt = DefaultTextPrompt, verbose = false, scale = 3, concurrency = 2, onProgress } = options;
await external_fs_extra_default().ensureDir(outputDir);
console.log("\u5F00\u59CB\u89E3\u6790PDF\u6587\u4EF6(\u5168\u9875\u6A21\u5F0F):", pdfPath);
try {
console.log("\u751F\u6210\u5168\u9875\u56FE\u50CF...");
const imageOutputDir = external_path_default().join(outputDir, 'pages');
await external_fs_extra_default().ensureDir(imageOutputDir);
const imageFiles = await generateFullPageImages(pdfPath, imageOutputDir, scale);
if (onProgress) onProgress({
current: 0,
total: imageFiles.length,
taskStatus: 'starting'
});
console.log("\u5904\u7406\u5168\u9875\u56FE\u50CF...");
const modelClient = new src_modelClient({
apiKey,
baseUrl,
model,
openAiApicompatible
});
const pageContents = [];
const processImages = async (item)=>{
console.log(`\u{5904}\u{7406}\u{9875}\u{9762} ${item.index}/${imageFiles.length}`);
try {
const defaultPrompt = "\u8BF7\u5C06\u56FE\u50CF\u4E2D\u7684\u6240\u6709\u6587\u672C\u5185\u5BB9\u8F6C\u6362\u4E3AMarkdown\u683C\u5F0F\uFF0C\u5305\u62EC\u6807\u9898\u3001\u6BB5\u843D\u3001\u5217\u8868\u548C\u8868\u683C\u7B49\u3002";
const pageContent = await modelClient.processImage(item.data, prompt || defaultPrompt);
pageContents.push({
pageIndex: item.index,
content: pageContent
});
if (onProgress) onProgress({
current: pageContents.length,
total: imageFiles.length,
taskStatus: 'running'
});
return {
success: true,
item,
data: pageContent
};
} catch (error) {
console.error("Markdown \u8F6C\u6362\u5931\u8D25:", error);
return {
success: false,
item,
error: error.message
};
}
};
await processInParallel(imageFiles, processImages, concurrency);
pageContents.sort((a, b)=>a.pageIndex - b.pageIndex);
console.log("\u751F\u6210Markdown\u6587\u6863...");
let content = '';
for (const page of pageContents)content += page.content + '\n';
console.log("\u6B63\u5728\u91CD\u65B0\u8C03\u6574\u76EE\u5F55...");
const title = await getOldMarkdownHeadings(content);
const defaultPrompt = "\u8BF7\u5C06\u56FE\u50CF\u4E2D\u7684\u6240\u6709\u6587\u672C\u5185\u5BB9\u8F6C\u6362\u4E3AMarkdown\u683C\u5F0F\uFF0C\u5305\u62EC\u6807\u9898\u3001\u6BB5\u843D\u3001\u5217\u8868\u548C\u8868\u683C\u7B49\u3002";
const convertedTitleLLMResult = await modelClient.processImage(null, textPrompt + JSON.stringify(title) || defaultPrompt);
const convertedTitle = await extractMdFromLLMOutput(convertedTitleLLMResult);
const convertContent = await adjustMarkdownHeadings(content, convertedTitle || '');
console.log("\u76EE\u5F55\u8C03\u6574\u5B8C\u6210...");
const mdFilePath = external_path_default().join(outputDir, external_path_default().basename(pdfPath, '.pdf') + '.md');
await external_fs_extra_default().writeFile(mdFilePath, convertContent);
console.log("Markdown\u6587\u4EF6\u5DF2\u4FDD\u5B58\u81F3:", mdFilePath);
if (onProgress) onProgress({
current: imageFiles.length,
total: imageFiles.length,
taskStatus: 'finished'
});
return {
content,
mdFilePath
};
} catch (error) {
console.error("PDF\u89E3\u6790\u8FC7\u7A0B\u4E2D\u53D1\u751F\u9519\u8BEF:", error);
throw error;
}
};
const processInParallel = async (items, processFunction, concurrencyLimit)=>{
const results = [];
const inProgress = new Set();
const queue = [
...items
];
while(queue.length > 0 || inProgress.size > 0){
while(inProgress.size < concurrencyLimit && queue.length > 0){
const item = queue.shift();
const promise = processFunction(item).then((result)=>{
inProgress.delete(promise);
return result;
});
inProgress.add(promise);
results.push(promise);
}
if (inProgress.size > 0) await Promise.race(inProgress);
}
return Promise.all(results);
};
const getPageNum = async (pdfData)=>{
const pdfPageCount = await getPageCount(pdfData);
return pdfPageCount;
};
exports.getPageNum = __webpack_exports__.getPageNum;
exports.parsePdf = __webpack_exports__.parsePdf;
for(var __webpack_i__ in __webpack_exports__)if (-1 === [
"getPageNum",
"parsePdf"
].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
Object.defineProperty(exports, '__esModule', {
value: true
});