apple-hig-mcp
Version:
High-performance MCP server providing instant access to Apple's Human Interface Guidelines via hybrid static/dynamic content delivery
626 lines (621 loc) • 25.1 kB
JavaScript
/**
* CrawleeHIGService
*
* Modern replacement for HIGScraper using Crawlee's PlaywrightCrawler
* for JavaScript-capable content extraction from Apple's SPA-based HIG website.
*
* Achieves 95%+ real content extraction by properly handling Apple's
* Single Page Application architecture.
*/
import { HIGDiscoveryService } from './hig-discovery.service.js';
export class CrawleeHIGService {
cache;
discoveryService;
config;
requestCount = 0;
lastRequestTime = 0;
constructor(cache) {
this.cache = cache;
this.discoveryService = new HIGDiscoveryService(cache);
// Suppress Crawlee/Playwright logging when not in development
if (process.env.NODE_ENV !== 'development') {
process.env.CRAWLEE_LOG_LEVEL = 'OFF';
process.env.CRAWLEE_VERBOSE_LOG = 'false';
process.env.APIFY_LOG_LEVEL = 'OFF';
process.env.PLAYWRIGHT_LOG_LEVEL = 'OFF';
}
this.config = {
// Base scraping config
baseUrl: 'https://developer.apple.com/design/human-interface-guidelines',
userAgent: 'Apple-HIG-MCP-Server/2.0.0 (Educational/Development Purpose; Crawlee-Powered)',
requestDelay: 2000, // 2 seconds between requests
retryAttempts: 3,
timeout: 15000,
// Crawlee-specific config
maxConcurrency: 2, // Conservative for Apple
browserOptions: {
headless: true,
viewport: { width: 1920, height: 1080 },
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--disable-web-security',
'--disable-features=VizDisplayCompositor',
'--log-level=3',
'--silent',
'--disable-logging',
'--disable-dev-tools',
'--disable-extensions-http-throttling'
]
},
waitOptions: {
networkIdle: 2000, // Wait 2s after network idle
timeout: 15000
}
};
}
/**
* Discover all HIG sections using dynamic discovery
*/
async discoverSections() {
return this.discoveryService.discoverSections();
}
/**
* Fetch content for a specific HIG section using Crawlee
*/
async fetchSectionContent(section) {
const cacheKey = `hig:crawlee:section:${section.id}`;
// Check cache first
const cached = this.cache.get(cacheKey);
if (cached) {
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Using cached content for: ${section.title}`);
}
return cached;
}
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Extracting content for: ${section.title}`);
}
try {
const extractionResult = await this.extractContentWithCrawlee(section.url);
const updatedSection = {
...section,
content: extractionResult.content,
lastUpdated: extractionResult.timestamp
};
// Cache successful extractions for 2 hours
this.cache.set(cacheKey, updatedSection, 7200);
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Successfully extracted content for: ${section.title} (quality: ${extractionResult.quality}, method: ${extractionResult.extractionMethod})`);
}
return updatedSection;
}
catch (error) {
if (process.env.NODE_ENV === 'development') {
console.error(`[CrawleeHIG] Failed to extract content for ${section.title}:`, error);
}
// Return section with minimal fallback content
return {
...section,
content: this.getMinimalFallbackContent(section),
lastUpdated: new Date()
};
}
}
/**
* Extract content using direct Playwright browser instead of Crawlee
*/
async extractContentWithCrawlee(url) {
// Respect rate limiting
await this.respectRateLimit();
// Use Playwright directly for more reliable single-page extraction
const { chromium } = await import('playwright');
let browser;
let page;
try {
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Processing: ${url}`);
}
// Launch browser with our configuration
browser = await chromium.launch({
headless: this.config.browserOptions.headless,
args: this.config.browserOptions.args
});
// Create a new page
page = await browser.newPage({
viewport: this.config.browserOptions.viewport
});
// Set user agent
await page.setExtraHTTPHeaders({
'User-Agent': this.config.userAgent
});
// Navigate to the URL
await page.goto(url, {
waitUntil: 'networkidle',
timeout: this.config.waitOptions.timeout
});
// Additional wait to ensure dynamic content is loaded
await page.waitForTimeout(this.config.waitOptions.networkIdle);
// Extract content from the page
const contentResult = await this.extractPageContent(page);
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Content extracted, length: ${contentResult.content.length}, quality: ${contentResult.quality}`);
}
return {
content: contentResult.content,
quality: contentResult.quality,
extractionMethod: 'crawlee',
timestamp: new Date()
};
}
catch (error) {
if (process.env.NODE_ENV === 'development') {
console.error(`[CrawleeHIG] Page processing error: ${error}`);
}
throw error;
}
finally {
// Clean up resources
if (page) {
await page.close().catch(() => { });
}
if (browser) {
await browser.close().catch(() => { });
}
}
}
/**
* Extract content from a loaded page
*/
async extractPageContent(page) {
try {
// Multiple strategies for content extraction
const strategies = [
() => this.extractWithMainContentStrategy(page),
() => this.extractWithArticleStrategy(page),
() => this.extractWithBodyStrategy(page),
() => this.extractWithFallbackStrategy(page)
];
let bestContent = '';
let bestQuality = 0;
for (const strategy of strategies) {
try {
const result = await strategy();
if (result.quality > bestQuality) {
bestContent = result.content;
bestQuality = result.quality;
}
}
catch (error) {
if (process.env.NODE_ENV === 'development') {
console.warn(`[CrawleeHIG] Content extraction strategy failed: ${error}`);
}
}
}
return {
content: bestContent,
quality: bestQuality
};
}
catch (error) {
if (process.env.NODE_ENV === 'development') {
console.error(`[CrawleeHIG] Content extraction failed: ${error}`);
}
throw error;
}
}
/**
* Extract content using main content strategy
*/
async extractWithMainContentStrategy(page) {
const selectors = [
'main',
'[role="main"]',
'.main-content',
'.content',
'.documentation-content',
'.article-content'
];
for (const selector of selectors) {
try {
const content = await page.$eval(selector, (element) => {
// Simple text extraction with basic markdown formatting
const extractText = (el) => {
let text = '';
for (const child of el.childNodes) {
if (child.nodeType === 3) { // Text node
text += child.textContent || '';
}
else if (child.nodeType === 1) { // Element node
const tagName = child.tagName?.toLowerCase();
switch (tagName) {
case 'h1':
case 'h2':
case 'h3':
case 'h4':
case 'h5':
case 'h6': {
const level = parseInt(tagName[1]);
text += '\n' + '#'.repeat(level) + ' ' + (child.textContent || '') + '\n\n';
break;
}
case 'p':
text += (child.textContent || '') + '\n\n';
break;
case 'li':
text += '- ' + (child.textContent || '') + '\n';
break;
case 'code':
text += '`' + (child.textContent || '') + '`';
break;
case 'strong':
case 'b':
text += '**' + (child.textContent || '') + '**';
break;
case 'em':
case 'i':
text += '*' + (child.textContent || '') + '*';
break;
default:
text += extractText(child);
}
}
}
return text;
};
return extractText(element);
});
if (content && content.length > 500) {
return {
content: this.cleanMarkdownContent(content),
quality: this.calculateContentQuality(content)
};
}
}
catch {
// Try next selector
}
}
throw new Error('Main content strategy failed');
}
/**
* Extract content using article strategy
*/
async extractWithArticleStrategy(page) {
const selectors = [
'article',
'.article',
'.documentation',
'.prose'
];
for (const selector of selectors) {
try {
const content = await page.$eval(selector, (element) => {
// Simple text extraction with basic markdown formatting
const extractText = (el) => {
let text = '';
for (const child of el.childNodes) {
if (child.nodeType === 3) { // Text node
text += child.textContent || '';
}
else if (child.nodeType === 1) { // Element node
const tagName = child.tagName?.toLowerCase();
switch (tagName) {
case 'h1':
case 'h2':
case 'h3':
case 'h4':
case 'h5':
case 'h6': {
const level = parseInt(tagName[1]);
text += '\n' + '#'.repeat(level) + ' ' + (child.textContent || '') + '\n\n';
break;
}
case 'p':
text += (child.textContent || '') + '\n\n';
break;
case 'li':
text += '- ' + (child.textContent || '') + '\n';
break;
case 'code':
text += '`' + (child.textContent || '') + '`';
break;
case 'strong':
case 'b':
text += '**' + (child.textContent || '') + '**';
break;
case 'em':
case 'i':
text += '*' + (child.textContent || '') + '*';
break;
default:
text += extractText(child);
}
}
}
return text;
};
return extractText(element);
});
if (content && content.length > 300) {
return {
content: this.cleanMarkdownContent(content),
quality: this.calculateContentQuality(content)
};
}
}
catch {
// Try next selector
}
}
throw new Error('Article strategy failed');
}
/**
* Extract content using body strategy
*/
async extractWithBodyStrategy(page) {
try {
const content = await page.evaluate(() => {
// Remove unwanted elements
const unwantedSelectors = [
'script', 'style', 'nav', 'header', 'footer',
'.navigation', '.sidebar', '.breadcrumb',
'.menu', '.toolbar', '.ad', '.advertisement'
];
unwantedSelectors.forEach(selector => {
// eslint-disable-next-line no-undef
const elements = document.querySelectorAll(selector);
elements.forEach((el) => el.remove());
});
// Convert body to markdown (simple text extraction)
// eslint-disable-next-line no-undef
return document.body?.textContent || '';
});
if (content && content.length > 200) {
return {
content: this.cleanMarkdownContent(content),
quality: this.calculateContentQuality(content) * 0.7 // Lower quality score for body strategy
};
}
}
catch {
// Continue to fallback
}
throw new Error('Body strategy failed');
}
/**
* Extract content using fallback strategy
*/
async extractWithFallbackStrategy(page) {
try {
const content = await page.evaluate(() => {
// eslint-disable-next-line no-undef
return document.body?.innerText || '';
});
if (content && content.length > 100) {
return {
content: content.trim(),
quality: 0.3 // Low quality for plain text
};
}
}
catch {
// Final fallback
}
throw new Error('All content extraction strategies failed');
}
/**
* Convert DOM element to markdown (to be injected into page context)
*/
convertElementToMarkdown(element) {
// This function will be injected into the page context
const convertElement = (el) => {
let markdown = '';
if (!el || !el.children)
return '';
for (const child of el.children) {
const tagName = child.tagName?.toLowerCase();
const text = child.textContent?.trim() || '';
if (!text)
continue;
switch (tagName) {
case 'h1':
markdown += `# ${text}\n\n`;
break;
case 'h2':
markdown += `## ${text}\n\n`;
break;
case 'h3':
markdown += `### ${text}\n\n`;
break;
case 'h4':
markdown += `#### ${text}\n\n`;
break;
case 'h5':
markdown += `##### ${text}\n\n`;
break;
case 'h6':
markdown += `###### ${text}\n\n`;
break;
case 'p':
markdown += `${text}\n\n`;
break;
case 'blockquote':
markdown += `> ${text}\n\n`;
break;
case 'pre':
case 'code':
markdown += `\`\`\`\n${text}\n\`\`\`\n\n`;
break;
case 'ul':
for (const li of child.children) {
if (li.tagName?.toLowerCase() === 'li') {
markdown += `- ${li.textContent?.trim()}\n`;
}
}
markdown += '\n';
break;
case 'ol': {
let index = 1;
for (const li of child.children) {
if (li.tagName?.toLowerCase() === 'li') {
markdown += `${index}. ${li.textContent?.trim()}\n`;
index++;
}
}
markdown += '\n';
break;
}
default:
// Recursively process child elements
markdown += convertElement(child);
}
}
return markdown;
};
return convertElement(element);
}
/**
* Clean and normalize markdown content
*/
cleanMarkdownContent(content) {
return content
.replace(/\n\s*\n\s*\n/g, '\n\n') // Remove excessive line breaks
.replace(/^\s+|\s+$/g, '') // Trim
.replace(/\t/g, ' ') // Convert tabs to spaces
.substring(0, 100000); // Limit content length
}
/**
* Calculate content quality score (0-1)
*/
calculateContentQuality(content) {
let score = 0;
// Length score (more content is generally better)
if (content.length > 1000)
score += 0.3;
else if (content.length > 500)
score += 0.2;
else if (content.length > 200)
score += 0.1;
// Structure score (markdown headers indicate good structure)
const headerCount = (content.match(/^#+\s/gm) || []).length;
if (headerCount > 5)
score += 0.2;
else if (headerCount > 2)
score += 0.1;
// Apple-specific content indicators
const appleTerms = [
'apple', 'ios', 'macos', 'watchos', 'tvos', 'visionos',
'human interface guidelines', 'design', 'interface',
'button', 'navigation', 'layout', 'accessibility'
];
const foundTerms = appleTerms.filter(term => content.toLowerCase().includes(term)).length;
score += Math.min(foundTerms * 0.05, 0.3);
// Fallback content detection (penalize generic content)
if (content.includes('This page requires JavaScript') ||
content.includes('single page application')) {
score = 0.1;
}
return Math.min(score, 1.0);
}
/**
* Respect rate limiting
*/
async respectRateLimit() {
const now = Date.now();
const timeSinceLastRequest = now - this.lastRequestTime;
if (timeSinceLastRequest < this.config.requestDelay) {
const waitTime = this.config.requestDelay - timeSinceLastRequest;
if (process.env.NODE_ENV === 'development') {
console.log(`[CrawleeHIG] Rate limiting: waiting ${waitTime}ms`);
}
await new Promise(resolve => setTimeout(resolve, waitTime));
}
this.lastRequestTime = Date.now();
this.requestCount++;
}
/**
* Get minimal fallback content when all extraction methods fail
*/
getMinimalFallbackContent(section) {
return `# ${section.title}
This section is part of Apple's Human Interface Guidelines.
**Note**: Content extraction failed. Please visit the official documentation for complete information:
${section.url}
---
**Attribution Notice**
This content is sourced from Apple's Human Interface Guidelines: ${section.url}
© Apple Inc. All rights reserved. For the most up-to-date and official information, please refer to Apple's official documentation.`;
}
/**
* Search content using existing discovery and extraction capabilities
*/
async searchContent(query, platform, category, limit = 10) {
const sections = await this.discoverSections();
let filteredSections = sections;
// Filter by platform and category
if (platform && platform !== 'universal') {
filteredSections = filteredSections.filter(s => s.platform === platform || s.platform === 'universal');
}
if (category) {
filteredSections = filteredSections.filter(s => s.category === category);
}
const results = [];
const queryLower = query.toLowerCase();
const queryWords = queryLower.split(/\s+/).filter(word => word.length > 2);
// Search through sections for matches
for (const section of filteredSections.slice(0, limit * 2)) {
let relevanceScore = 0;
let matchReason = '';
// Check title matches
const titleLower = section.title.toLowerCase();
if (titleLower.includes(queryLower)) {
relevanceScore += 2.0;
matchReason = 'exact title match';
}
else {
const titleWordMatches = queryWords.filter(word => titleLower.includes(word)).length;
if (titleWordMatches > 0) {
relevanceScore += titleWordMatches * 0.8;
matchReason = 'partial title match';
}
}
// Check URL matches
const urlLower = section.url.toLowerCase();
if (urlLower.includes(queryLower)) {
relevanceScore += 1.5;
matchReason = matchReason || 'URL match';
}
// Add to results if relevant
if (relevanceScore > 0) {
results.push({
id: section.id,
title: section.title,
url: section.url,
platform: section.platform,
relevanceScore,
snippet: `${section.title} - ${matchReason}`,
type: 'section'
});
}
}
// Sort by relevance and limit results
return results
.sort((a, b) => b.relevanceScore - a.relevanceScore)
.slice(0, limit);
}
/**
* Clean up service resources
*/
async teardown() {
// Clear cache if needed
this.cache.clear();
// Reset counters
this.requestCount = 0;
this.lastRequestTime = 0;
}
}
//# sourceMappingURL=crawlee-hig.service.js.map