defect-inspection-tools-mcp-server
Version:
Defect inspection tools server handling defect detection, analysis, and reporting with AI/ML capabilities for quality control
370 lines • 14.3 kB
JavaScript
import { logger } from './logger.js';
import { SanitizationMiddleware } from '../middleware/sanitization-middleware.js';
export class DocumentParser {
constructor(externalApiService) {
this.externalApiService = externalApiService;
}
// Parse document from URL
async parseDocumentFromUrl(url, options = {}) {
try {
const opts = { ...DocumentParser.DEFAULT_OPTIONS, ...options };
logger.info('Starting document parsing from URL', {
url: SanitizationMiddleware.sanitizeForLogging(url),
options: opts
});
// Validate URL
const sanitizedUrl = SanitizationMiddleware.sanitizeFilePath(url);
if (!this.isValidUrl(sanitizedUrl)) {
throw new Error('Invalid URL provided');
}
// Fetch document
const response = await this.externalApiService.get(sanitizedUrl, {}, {
timeout: opts.timeout
});
if (!response.success) {
throw new Error(`Failed to fetch document: ${response.error}`);
}
const contentType = response.data?.headers?.['content-type'] || '';
const fileSize = response.data?.headers?.['content-length'] || 0;
// Check MIME type
if (opts.allowedMimeTypes && !this.isAllowedMimeType(contentType, opts.allowedMimeTypes)) {
throw new Error(`Unsupported MIME type: ${contentType}`);
}
// Parse based on content type
let parseResult;
if (contentType.includes('application/pdf')) {
parseResult = await this.parsePdf(response.data?.data || '', opts);
}
else if (contentType.includes('text/html')) {
parseResult = await this.parseHtml(response.data?.data || '', opts);
}
else if (contentType.includes('text/markdown')) {
parseResult = await this.parseMarkdown(response.data?.data || '', opts);
}
else if (contentType.includes('text/plain')) {
parseResult = await this.parseText(response.data?.data || '', opts);
}
else {
// Default to text parsing
parseResult = await this.parseText(response.data?.data || '', opts);
}
// Add metadata
parseResult.metadata = {
...parseResult.metadata,
fileSize: Number(fileSize),
mimeType: contentType,
extractedAt: new Date()
};
// Sanitize result
const sanitizedResult = SanitizationMiddleware.sanitizeToolResponse(parseResult);
logger.info('Document parsing completed successfully', {
url: SanitizationMiddleware.sanitizeForLogging(url),
contentLength: parseResult.content.length,
mimeType: contentType,
sectionsCount: parseResult.sections?.length || 0,
tablesCount: parseResult.tables?.length || 0,
linksCount: parseResult.links?.length || 0
});
return sanitizedResult;
}
catch (error) {
logger.error('Document parsing failed', {
url: SanitizationMiddleware.sanitizeForLogging(url),
error: error.message
});
throw error;
}
}
// Parse PDF content
async parsePdf(content, options) {
try {
// For now, return basic structure
// In production, you would integrate with a PDF parsing library like pdf-parse
const textContent = this.extractTextFromPdf(content);
return {
content: textContent.substring(0, options.maxContentLength),
metadata: {
title: 'PDF Document',
extractedAt: new Date()
},
sections: options.extractSections ? this.extractSections(textContent) : undefined,
tables: options.extractTables ? this.extractTablesFromText(textContent) : undefined,
links: options.extractLinks ? this.extractLinksFromText(textContent) : undefined
};
}
catch (error) {
logger.error('PDF parsing failed', { error: error.message });
throw new Error('Failed to parse PDF document');
}
}
// Parse HTML content
async parseHtml(content, options) {
try {
const textContent = this.stripHtmlTags(content);
const title = this.extractHtmlTitle(content);
return {
content: textContent.substring(0, options.maxContentLength),
metadata: {
title,
extractedAt: new Date()
},
sections: options.extractSections ? this.extractHtmlSections(content) : undefined,
tables: options.extractTables ? this.extractHtmlTables(content) : undefined,
links: options.extractLinks ? this.extractHtmlLinks(content) : undefined
};
}
catch (error) {
logger.error('HTML parsing failed', { error: error.message });
throw new Error('Failed to parse HTML document');
}
}
// Parse Markdown content
async parseMarkdown(content, options) {
try {
const textContent = this.stripMarkdownFormatting(content);
const title = this.extractMarkdownTitle(content);
return {
content: textContent.substring(0, options.maxContentLength),
metadata: {
title,
extractedAt: new Date()
},
sections: options.extractSections ? this.extractMarkdownSections(content) : undefined,
tables: options.extractTables ? this.extractMarkdownTables(content) : undefined,
links: options.extractLinks ? this.extractMarkdownLinks(content) : undefined
};
}
catch (error) {
logger.error('Markdown parsing failed', { error: error.message });
throw new Error('Failed to parse Markdown document');
}
}
// Parse plain text content
async parseText(content, options) {
try {
const textContent = String(content);
return {
content: textContent.substring(0, options.maxContentLength),
metadata: {
title: 'Plain Text Document',
extractedAt: new Date()
},
sections: options.extractSections ? this.extractSections(textContent) : undefined,
tables: options.extractTables ? this.extractTablesFromText(textContent) : undefined,
links: options.extractLinks ? this.extractLinksFromText(textContent) : undefined
};
}
catch (error) {
logger.error('Text parsing failed', { error: error.message });
throw new Error('Failed to parse text document');
}
}
// Helper methods
isValidUrl(url) {
try {
new URL(url);
return true;
}
catch {
return false;
}
}
isAllowedMimeType(contentType, allowedTypes) {
return allowedTypes.some(type => contentType.includes(type));
}
extractTextFromPdf(content) {
// Placeholder for PDF text extraction
// In production, integrate with pdf-parse or similar library
return 'PDF content extraction not implemented';
}
stripHtmlTags(html) {
return html.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim();
}
extractHtmlTitle(html) {
const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
return titleMatch ? titleMatch[1].trim() : 'Untitled Document';
}
extractHtmlSections(html) {
const sections = [];
const headingRegex = /<h([1-6])[^>]*>([^<]+)<\/h[1-6]>/gi;
let match;
while ((match = headingRegex.exec(html)) !== null) {
sections.push({
heading: match[2].trim(),
content: '',
level: parseInt(match[1])
});
}
return sections;
}
extractHtmlTables(html) {
const tables = [];
const tableRegex = /<table[^>]*>([\s\S]*?)<\/table>/gi;
let match;
while ((match = tableRegex.exec(html)) !== null) {
const tableContent = match[1];
const headers = this.extractTableHeaders(tableContent);
const rows = this.extractTableRows(tableContent);
if (headers.length > 0 || rows.length > 0) {
tables.push({ headers, rows });
}
}
return tables;
}
extractHtmlLinks(html) {
const links = [];
const linkRegex = /<a[^>]*href=['"]([^'"]+)['"][^>]*>([^<]+)<\/a>/gi;
let match;
while ((match = linkRegex.exec(html)) !== null) {
links.push({
url: match[1],
text: match[2].trim()
});
}
return links;
}
stripMarkdownFormatting(markdown) {
return markdown
.replace(/^#+\s*/gm, '') // Remove headers
.replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
.replace(/\*([^*]+)\*/g, '$1') // Remove italic
.replace(/`([^`]+)`/g, '$1') // Remove code
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Remove links
.replace(/^\s*[-*+]\s*/gm, '') // Remove list markers
.replace(/\s+/g, ' ')
.trim();
}
extractMarkdownTitle(markdown) {
const titleMatch = markdown.match(/^#\s+(.+)$/m);
return titleMatch ? titleMatch[1].trim() : 'Untitled Document';
}
extractMarkdownSections(markdown) {
const sections = [];
const headingRegex = /^(#{1,6})\s+(.+)$/gm;
let match;
while ((match = headingRegex.exec(markdown)) !== null) {
sections.push({
heading: match[2].trim(),
content: '',
level: match[1].length
});
}
return sections;
}
extractMarkdownTables(markdown) {
const tables = [];
const tableRegex = /\|(.+)\|\s*\n\|[-\s|:]+\|\s*\n((?:\|.+\|\s*\n?)*)/gm;
let match;
while ((match = tableRegex.exec(markdown)) !== null) {
const headers = match[1].split('|').map(h => h.trim()).filter(h => h);
const rowsText = match[2];
const rows = rowsText.split('\n')
.filter(row => row.trim())
.map(row => row.split('|').map(cell => cell.trim()).filter(cell => cell));
if (headers.length > 0 && rows.length > 0) {
tables.push({ headers, rows });
}
}
return tables;
}
extractMarkdownLinks(markdown) {
const links = [];
const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
let match;
while ((match = linkRegex.exec(markdown)) !== null) {
links.push({
url: match[2],
text: match[1]
});
}
return links;
}
extractSections(text) {
const sections = [];
const lines = text.split('\n');
for (const line of lines) {
if (line.trim() && (line.toUpperCase() === line || line.includes(':'))) {
sections.push({
heading: line.trim(),
content: '',
level: 1
});
}
}
return sections;
}
extractTablesFromText(text) {
const tables = [];
const lines = text.split('\n');
// Look for lines that might be table-like (contains multiple delimiters)
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.includes('\t') || line.includes('|') || line.includes(',')) {
const delimiter = line.includes('\t') ? '\t' : line.includes('|') ? '|' : ',';
const cells = line.split(delimiter).map(cell => cell.trim());
if (cells.length > 1) {
tables.push({
headers: cells,
rows: []
});
}
}
}
return tables;
}
extractLinksFromText(text) {
const links = [];
const urlRegex = /(https?:\/\/[^\s]+)/g;
let match;
while ((match = urlRegex.exec(text)) !== null) {
links.push({
url: match[1],
text: match[1]
});
}
return links;
}
extractTableHeaders(tableContent) {
const headerRegex = /<th[^>]*>([^<]+)<\/th>/gi;
const headers = [];
let match;
while ((match = headerRegex.exec(tableContent)) !== null) {
headers.push(match[1].trim());
}
return headers;
}
extractTableRows(tableContent) {
const rows = [];
const rowRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
let rowMatch;
while ((rowMatch = rowRegex.exec(tableContent)) !== null) {
const rowContent = rowMatch[1];
const cellRegex = /<td[^>]*>([^<]+)<\/td>/gi;
const cells = [];
let cellMatch;
while ((cellMatch = cellRegex.exec(rowContent)) !== null) {
cells.push(cellMatch[1].trim());
}
if (cells.length > 0) {
rows.push(cells);
}
}
return rows;
}
}
DocumentParser.DEFAULT_OPTIONS = {
maxContentLength: 50000,
extractTables: true,
extractLinks: true,
extractSections: true,
allowedMimeTypes: [
'application/pdf',
'text/plain',
'text/html',
'text/markdown',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
],
timeout: 30000
};
//# sourceMappingURL=documentParser.js.map