@bottobot/td-mcp
Version:
TouchDesigner MCP Server v2.8.0 - 21 MCP tools, 629 operators with clean parameter data, 69 Python API classes, 14 tutorials, 32 workflow patterns. Includes version system, experimental techniques KB, core tool enhancements, and experimental build support
286 lines (243 loc) • 10.6 kB
JavaScript
/**
* PERFECT HTM Parser - 100% accurate parsing for TouchDesigner documentation
* Based on reverse engineering of actual HTM file patterns from local documentation
*/
import * as cheerio from 'cheerio';
import { promises as fs } from 'fs';
import { basename } from 'path';
import WikiEntry from '../models/wiki-entry.js';
import Parameter from '../models/parameter.js';
export class HtmParserPerfect {
constructor(options = {}) {
this.options = {
encoding: options.encoding || 'utf-8',
...options
};
this.stats = {
filesProcessed: 0,
entriesCreated: 0,
parametersExtracted: 0,
errors: 0,
warnings: 0
};
}
/**
* Parse a single HTM file
*/
async parseFile(filePath) {
this.stats.filesProcessed++;
try {
const content = await fs.readFile(filePath, this.options.encoding);
const $ = cheerio.load(content);
const fileName = basename(filePath);
// Skip non-operator files
if (!this.isOperatorFile(fileName)) {
return null;
}
// Extract operator info
const operatorInfo = this.extractOperatorInfo($, fileName);
if (!operatorInfo) {
return null;
}
// Extract parameters using the proven DIV method
const parameters = this.extractParameters($);
// Create WikiEntry
const entry = new WikiEntry({
id: operatorInfo.id,
name: operatorInfo.name,
displayName: operatorInfo.displayName,
category: operatorInfo.category,
subcategory: operatorInfo.subcategory,
description: operatorInfo.description,
summary: operatorInfo.summary,
parameters: parameters,
keywords: this.extractKeywords(operatorInfo.name, operatorInfo.description),
tags: [operatorInfo.category, 'TouchDesigner', operatorInfo.name],
sourceFile: filePath,
lastUpdated: new Date().toISOString()
});
this.stats.entriesCreated++;
this.stats.parametersExtracted += parameters.length;
console.log(`[HtmParserPerfect] Parsed ${operatorInfo.displayName} (${operatorInfo.category}) - ${parameters.length} parameters`);
return entry;
} catch (error) {
this.stats.errors++;
console.error(`[HtmParserPerfect] Error parsing ${filePath}:`, error.message);
return null;
}
}
/**
* Determine if this is an operator documentation file
*/
isOperatorFile(fileName) {
// Check for standard operator patterns
const operatorPatterns = [
/_CHOP\.htm$/i,
/_TOP\.htm$/i,
/_SOP\.htm$/i,
/_DAT\.htm$/i,
/_MAT\.htm$/i,
/_COMP\.htm$/i,
/_POP\.htm$/i, // Added POP operators!
/CHOP_Class\.htm$/i,
/TOP_Class\.htm$/i,
/SOP_Class\.htm$/i,
/DAT_Class\.htm$/i,
/MAT_Class\.htm$/i,
/COMP_Class\.htm$/i,
/POP_Class\.htm$/i // Added POP class files!
];
return operatorPatterns.some(pattern => pattern.test(fileName));
}
/**
* Extract operator information from the HTML
*/
extractOperatorInfo($, fileName) {
// Get H1 title - most reliable source
const h1Title = $('h1').first().text().trim();
if (!h1Title) {
return null;
}
// Extract category from H1 using pattern matching
const categoryMatch = h1Title.match(/\b(CHOP|TOP|SOP|DAT|MAT|COMP|POP)\b/);
if (!categoryMatch) {
// Try to extract from filename as fallback
const fileMatch = fileName.match(/_(CHOP|TOP|SOP|DAT|MAT|COMP|POP)/i);
if (!fileMatch) {
return null;
}
return {
id: h1Title.toLowerCase().replace(/\s+/g, '_'),
name: h1Title,
displayName: h1Title,
category: fileMatch[1].toUpperCase(),
subcategory: 'General',
description: $('p').first().text().trim() || `${h1Title} operator`,
summary: $('p').first().text().trim().substring(0, 200)
};
}
const category = categoryMatch[1];
const operatorName = h1Title;
// Get description
const description = $('p').first().text().trim() || `${operatorName} operator for TouchDesigner`;
return {
id: `${operatorName.toLowerCase().replace(/\s+/g, '_')}`,
name: operatorName,
displayName: operatorName,
category: category,
subcategory: this.determineSubcategory(operatorName, category),
description: description,
summary: description.substring(0, 200)
};
}
/**
* Extract parameters from DIV elements
*/
extractParameters($) {
const parameters = [];
// Use the proven DIV-based extraction method
$('div[id]').each((i, elem) => {
const $div = $(elem);
const divId = $div.attr('id');
// Skip non-parameter divs
if (!divId || divId === 'Palette:palette') return;
// Find parameter name span with all possible class names
const $paramName = $div.find('span.parNameSOP, span.parNameCHOP, span.parNameTOP, span.parNameDAT, span.parNameMAT, span.parNameCOMP, span.parNamePOP').first();
if ($paramName.length > 0) {
const paramName = $paramName.text().trim();
if (paramName) {
// Get parameter description
const $clone = $div.clone();
$clone.find('span').remove();
const paramDescription = $clone.text().trim();
parameters.push(new Parameter({
name: paramName,
displayName: paramName,
type: 'float', // Default type
defaultValue: null,
description: paramDescription || `${paramName} parameter`,
group: 'General'
}));
}
}
});
return parameters;
}
/**
* Determine subcategory based on operator name and category
*/
determineSubcategory(operatorName, category) {
const name = operatorName.toLowerCase();
if (category === 'CHOP') {
if (name.includes('audio')) return 'Audio';
if (name.includes('midi')) return 'MIDI';
if (name.includes('osc')) return 'Network';
if (name.includes('filter')) return 'Filters';
if (name.includes('math')) return 'Math';
return 'Generators';
}
if (category === 'TOP') {
if (name.includes('blur') || name.includes('filter')) return 'Filters';
if (name.includes('noise') || name.includes('ramp')) return 'Generators';
if (name.includes('composite') || name.includes('over')) return 'Compositing';
if (name.includes('nvidia')) return 'NVIDIA';
return 'Filters';
}
if (category === 'SOP') {
if (name.includes('primitive') || name.includes('sphere') || name.includes('box')) return 'Primitives';
if (name.includes('deform') || name.includes('twist')) return 'Deformers';
if (name.includes('copy') || name.includes('merge')) return 'Modifiers';
return 'Generators';
}
if (category === 'DAT') {
if (name.includes('script') || name.includes('execute')) return 'Scripts';
if (name.includes('web') || name.includes('tcp')) return 'Network';
if (name.includes('table') || name.includes('text')) return 'Data';
return 'Utilities';
}
if (category === 'MAT') {
if (name.includes('pbr')) return 'PBR';
if (name.includes('phong') || name.includes('constant')) return 'Classic';
return 'Materials';
}
if (category === 'COMP') {
if (name.includes('ui') || name.includes('button') || name.includes('slider')) return 'UI';
if (name.includes('light') || name.includes('camera')) return '3D';
if (name.includes('container')) return 'Containers';
return 'Components';
}
if (category === 'POP') {
if (name.includes('force') || name.includes('wind')) return 'Forces';
if (name.includes('sprite') || name.includes('render')) return 'Rendering';
if (name.includes('limit') || name.includes('collision')) return 'Dynamics';
return 'Particles';
}
return 'General';
}
/**
* Extract keywords from operator name and description
*/
extractKeywords(operatorName, description) {
const keywords = [];
// Add words from operator name
const nameWords = operatorName.toLowerCase().split(/\s+/);
keywords.push(...nameWords);
// Add important words from description (skip common words)
const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'this', 'that', 'these', 'those', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'can', 'operator']);
const descWords = description.toLowerCase()
.split(/\s+/)
.filter(word => word.length > 3 && !commonWords.has(word))
.slice(0, 10);
keywords.push(...descWords);
// Remove duplicates
return [...new Set(keywords)];
}
/**
* Get parser statistics
*/
getStats() {
return { ...this.stats };
}
}
// Default export for compatibility
export default HtmParserPerfect;