UNPKG

img-to-text-computational

Version:

High-performance image-to-text analyzer using pure computational methods. Convert images to structured text descriptions with 99.9% accuracy, zero AI dependencies, and complete offline processing.

962 lines (809 loc) 30.2 kB
class ComponentRelationshipMapper { constructor(options = {}) { this.options = { proximityThreshold: options.proximityThreshold || 50, alignmentTolerance: options.alignmentTolerance || 10, containmentPadding: options.containmentPadding || 5, functionalRelationshipThreshold: options.functionalRelationshipThreshold || 0.6, ...options }; // Define functional relationship patterns this.functionalPatterns = this.initializeFunctionalPatterns(); } /** * Initialize functional relationship patterns */ initializeFunctionalPatterns() { return { form_relationships: [ { primary: 'input', secondary: 'button', relationship: 'form_submission' }, { primary: 'input', secondary: 'input', relationship: 'form_group' }, { primary: 'label', secondary: 'input', relationship: 'form_labeling' } ], navigation_relationships: [ { primary: 'navigation', secondary: 'navigation', relationship: 'nav_group' }, { primary: 'button', secondary: 'navigation', relationship: 'nav_action' } ], content_relationships: [ { primary: 'header', secondary: 'paragraph', relationship: 'content_hierarchy' }, { primary: 'image', secondary: 'text', relationship: 'media_caption' }, { primary: 'card', secondary: 'card', relationship: 'card_group' } ], layout_relationships: [ { primary: 'container', secondary: '*', relationship: 'containment' }, { primary: 'sidebar', secondary: 'main', relationship: 'layout_complement' } ] }; } /** * Map all component relationships * @param {Array} components - Array of detected components * @param {Object} layoutAnalysis - Layout analysis results * @param {Object} textElements - Text extraction results * @returns {Promise<Object>} Complete relationship mapping */ async mapComponentRelationships(components, layoutAnalysis, textElements) { try { const relationships = { spatial_relationships: await this.analyzeSpatialRelationships(components), functional_relationships: await this.analyzeFunctionalRelationships(components, textElements), hierarchical_relationships: await this.analyzeHierarchicalRelationships(components, layoutAnalysis), semantic_relationships: await this.analyzeSemanticRelationships(components, textElements), layout_relationships: await this.analyzeLayoutRelationships(components, layoutAnalysis), relationship_graph: this.buildRelationshipGraph(components), interaction_flows: await this.analyzeInteractionFlows(components, textElements), component_groups: await this.identifyComponentGroups(components), summary: {} }; // Generate relationship summary relationships.summary = this.generateRelationshipSummary(relationships); return relationships; } catch (error) { throw new Error(`Component relationship mapping failed: ${error.message}`); } } /** * Analyze spatial relationships between components */ async analyzeSpatialRelationships(components) { const spatialRelationships = []; for (let i = 0; i < components.length; i++) { for (let j = i + 1; j < components.length; j++) { const relationship = this.analyzeSpatialPair(components[i], components[j]); if (relationship.strength > 0.3) { spatialRelationships.push(relationship); } } } return { total_relationships: spatialRelationships.length, strong_relationships: spatialRelationships.filter(r => r.strength > 0.7).length, relationships: spatialRelationships.sort((a, b) => b.strength - a.strength), spatial_patterns: this.identifySpatialPatterns(spatialRelationships) }; } /** * Analyze spatial relationship between two components */ analyzeSpatialPair(comp1, comp2) { const relationship = { component1: comp1.id, component2: comp2.id, type: 'spatial', subtype: 'unknown', strength: 0, distance: 0, characteristics: [] }; // Calculate various distance metrics const distances = this.calculateDistances(comp1.position, comp2.position); relationship.distance = distances.center_to_center; // Analyze containment if (this.isContained(comp1.position, comp2.position)) { relationship.subtype = 'contained_by'; relationship.strength = 0.9; relationship.characteristics.push('comp1 contained in comp2'); return relationship; } if (this.isContained(comp2.position, comp1.position)) { relationship.subtype = 'contains'; relationship.strength = 0.9; relationship.characteristics.push('comp1 contains comp2'); return relationship; } // Analyze alignment const alignment = this.analyzeAlignment(comp1.position, comp2.position); if (alignment.horizontal_aligned && distances.center_to_center < 200) { relationship.subtype = 'horizontal_aligned'; relationship.strength = 0.7 - (distances.center_to_center / 500); relationship.characteristics.push('horizontally aligned'); } if (alignment.vertical_aligned && distances.center_to_center < 200) { relationship.subtype = 'vertical_aligned'; relationship.strength = 0.7 - (distances.center_to_center / 500); relationship.characteristics.push('vertically aligned'); } // Analyze adjacency if (distances.edge_to_edge < this.options.proximityThreshold) { relationship.subtype = 'adjacent'; relationship.strength = Math.max(relationship.strength, 0.6 - (distances.edge_to_edge / 100)); relationship.characteristics.push('adjacent components'); } // Analyze overlap if (this.hasOverlap(comp1.position, comp2.position)) { relationship.subtype = 'overlapping'; relationship.strength = 0.8; relationship.characteristics.push('overlapping components'); } // Analyze relative positioning const relativePosition = this.getRelativePosition(comp1.position, comp2.position); relationship.relative_position = relativePosition; relationship.characteristics.push(`comp2 is ${relativePosition} of comp1`); return relationship; } /** * Analyze functional relationships */ async analyzeFunctionalRelationships(components, textElements) { const functionalRelationships = []; // Check each functional pattern for (const [category, patterns] of Object.entries(this.functionalPatterns)) { for (const pattern of patterns) { const matches = this.findFunctionalMatches(pattern, components, textElements); functionalRelationships.push(...matches); } } // Analyze form relationships specifically const formRelationships = await this.analyzeFormRelationships(components, textElements); functionalRelationships.push(...formRelationships); // Analyze navigation relationships const navRelationships = await this.analyzeNavigationRelationships(components, textElements); functionalRelationships.push(...navRelationships); return { total_relationships: functionalRelationships.length, by_category: this.groupRelationshipsByCategory(functionalRelationships), relationships: functionalRelationships.sort((a, b) => b.confidence - a.confidence) }; } /** * Find functional pattern matches */ findFunctionalMatches(pattern, components, textElements) { const matches = []; const primaryComponents = components.filter(comp => pattern.primary === '*' || comp.type === pattern.primary ); const secondaryComponents = components.filter(comp => pattern.secondary === '*' || comp.type === pattern.secondary ); for (const primary of primaryComponents) { for (const secondary of secondaryComponents) { if (primary.id === secondary.id) continue; const confidence = this.calculateFunctionalConfidence( primary, secondary, pattern, textElements ); if (confidence > this.options.functionalRelationshipThreshold) { matches.push({ component1: primary.id, component2: secondary.id, type: 'functional', subtype: pattern.relationship, confidence, pattern, evidence: this.getFunctionalEvidence(primary, secondary, pattern, textElements) }); } } } return matches; } /** * Calculate functional relationship confidence */ calculateFunctionalConfidence(comp1, comp2, pattern, textElements) { let confidence = 0; // Base confidence from type matching if (comp1.type === pattern.primary && comp2.type === pattern.secondary) { confidence += 0.4; } // Proximity bonus const distance = this.calculateDistances(comp1.position, comp2.position).center_to_center; if (distance < 100) { confidence += 0.3; } else if (distance < 200) { confidence += 0.2; } // Text content analysis const textConfidence = this.analyzeTextualRelationship(comp1, comp2, pattern, textElements); confidence += textConfidence * 0.3; return Math.min(confidence, 1.0); } /** * Analyze form relationships specifically */ async analyzeFormRelationships(components, textElements) { const formRelationships = []; const inputs = components.filter(comp => comp.type === 'input'); const buttons = components.filter(comp => comp.type === 'button'); const labels = textElements?.structured_text?.filter(text => text.type === 'label' || text.text.endsWith(':') ) || []; // Input to button relationships (form submission) for (const input of inputs) { for (const button of buttons) { const distance = this.calculateDistances(input.position, button.position).center_to_center; if (distance < 300) { const isSubmitButton = button.text_content && /submit|send|save|register|login|sign|continue/i.test(button.text_content); const confidence = isSubmitButton ? 0.8 : 0.5; formRelationships.push({ component1: input.id, component2: button.id, type: 'functional', subtype: 'form_submission', confidence: confidence - (distance / 1000), evidence: [`Input field linked to ${isSubmitButton ? 'submit' : 'action'} button`] }); } } } // Label to input relationships for (const label of labels) { for (const input of inputs) { const distance = this.calculateDistances( { x: label.position.x, y: label.position.y, width: label.position.width, height: label.position.height }, input.position ).center_to_center; if (distance < 100) { formRelationships.push({ component1: `label_${label.id}`, component2: input.id, type: 'functional', subtype: 'form_labeling', confidence: 0.9 - (distance / 200), evidence: ['Label positioned near input field'] }); } } } // Input to input relationships (form groups) for (let i = 0; i < inputs.length; i++) { for (let j = i + 1; j < inputs.length; j++) { const distance = this.calculateDistances(inputs[i].position, inputs[j].position).center_to_center; if (distance < 150) { formRelationships.push({ component1: inputs[i].id, component2: inputs[j].id, type: 'functional', subtype: 'form_group', confidence: 0.7 - (distance / 300), evidence: ['Adjacent input fields in form'] }); } } } return formRelationships; } /** * Analyze navigation relationships */ async analyzeNavigationRelationships(components, textElements) { const navRelationships = []; const navComponents = components.filter(comp => comp.type === 'navigation' || comp.type === 'button' ); // Group navigation elements const navGroups = this.groupNavigationElements(navComponents); for (const group of navGroups) { if (group.length >= 2) { for (let i = 0; i < group.length; i++) { for (let j = i + 1; j < group.length; j++) { navRelationships.push({ component1: group[i].id, component2: group[j].id, type: 'functional', subtype: 'navigation_group', confidence: 0.8, evidence: ['Part of navigation group'] }); } } } } return navRelationships; } /** * Group navigation elements by proximity and alignment */ groupNavigationElements(navComponents) { const groups = []; const used = new Set(); for (const comp of navComponents) { if (used.has(comp.id)) continue; const group = [comp]; used.add(comp.id); // Find nearby navigation elements for (const other of navComponents) { if (used.has(other.id)) continue; const distance = this.calculateDistances(comp.position, other.position).center_to_center; const alignment = this.analyzeAlignment(comp.position, other.position); if (distance < 200 && (alignment.horizontal_aligned || alignment.vertical_aligned)) { group.push(other); used.add(other.id); } } if (group.length > 1) { groups.push(group); } } return groups; } /** * Analyze hierarchical relationships */ async analyzeHierarchicalRelationships(components, layoutAnalysis) { const hierarchicalRelationships = []; // Analyze containment hierarchies for (const container of components) { const contained = components.filter(comp => comp.id !== container.id && this.isContained(comp.position, container.position) ); for (const child of contained) { hierarchicalRelationships.push({ parent: container.id, child: child.id, type: 'hierarchical', subtype: 'containment', level: this.calculateHierarchyLevel(child, components), confidence: 0.9 }); } } // Analyze size-based hierarchy const sizeHierarchy = this.analyzeSizeHierarchy(components); hierarchicalRelationships.push(...sizeHierarchy); // Analyze z-index implications const zIndexHierarchy = this.analyzeZIndexHierarchy(components); hierarchicalRelationships.push(...zIndexHierarchy); return { total_relationships: hierarchicalRelationships.length, max_depth: this.calculateMaxHierarchyDepth(hierarchicalRelationships), relationships: hierarchicalRelationships }; } /** * Analyze semantic relationships */ async analyzeSemanticRelationships(components, textElements) { const semanticRelationships = []; // Header to content relationships const headers = textElements?.structured_text?.filter(text => text.type === 'header') || []; const contentElements = components.filter(comp => comp.type === 'paragraph' || comp.type === 'text' || comp.type === 'card' ); for (const header of headers) { for (const content of contentElements) { const distance = this.calculateDistances( { x: header.position.x, y: header.position.y, width: header.position.width, height: header.position.height }, content.position ).center_to_center; if (distance < 200 && header.position.y < content.position.y) { semanticRelationships.push({ component1: `header_${header.id}`, component2: content.id, type: 'semantic', subtype: 'header_content', confidence: 0.8 - (distance / 400), evidence: ['Header positioned above content'] }); } } } // Image to caption relationships const images = components.filter(comp => comp.type === 'image'); const textBlocks = textElements?.structured_text?.filter(text => text.text.length > 10 && text.text.length < 200 ) || []; for (const image of images) { for (const text of textBlocks) { const distance = this.calculateDistances( image.position, { x: text.position.x, y: text.position.y, width: text.position.width, height: text.position.height } ).center_to_center; if (distance < 150) { semanticRelationships.push({ component1: image.id, component2: `text_${text.id}`, type: 'semantic', subtype: 'image_caption', confidence: 0.7 - (distance / 300), evidence: ['Text positioned near image, likely caption'] }); } } } return { total_relationships: semanticRelationships.length, relationships: semanticRelationships.sort((a, b) => b.confidence - a.confidence) }; } /** * Analyze layout relationships */ async analyzeLayoutRelationships(components, layoutAnalysis) { const layoutRelationships = []; // Grid relationships if (layoutAnalysis.grid_analysis?.detected) { const gridRelationships = this.analyzeGridRelationships(components, layoutAnalysis.grid_analysis); layoutRelationships.push(...gridRelationships); } // Flexbox relationships if (layoutAnalysis.layout_type === 'flexbox') { const flexRelationships = this.analyzeFlexboxRelationships(components, layoutAnalysis); layoutRelationships.push(...flexRelationships); } // Alignment relationships const alignmentGroups = layoutAnalysis.alignment_analysis || {}; if (alignmentGroups.horizontal_groups) { for (const group of alignmentGroups.horizontal_groups) { if (group.elements.length >= 2) { for (let i = 0; i < group.elements.length; i++) { for (let j = i + 1; j < group.elements.length; j++) { layoutRelationships.push({ component1: group.elements[i].id, component2: group.elements[j].id, type: 'layout', subtype: 'horizontal_alignment', confidence: 0.8, evidence: ['Horizontally aligned in layout'] }); } } } } } return { total_relationships: layoutRelationships.length, by_subtype: this.groupRelationshipsBySubtype(layoutRelationships), relationships: layoutRelationships }; } /** * Analyze grid relationships */ analyzeGridRelationships(components, gridAnalysis) { const gridRelationships = []; // Assign components to grid cells const gridAssignments = this.assignComponentsToGrid(components, gridAnalysis); // Find grid neighbors for (const [compId, cell] of Object.entries(gridAssignments)) { for (const [otherCompId, otherCell] of Object.entries(gridAssignments)) { if (compId === otherCompId) continue; const isNeighbor = this.areGridNeighbors(cell, otherCell); if (isNeighbor) { gridRelationships.push({ component1: compId, component2: otherCompId, type: 'layout', subtype: 'grid_neighbors', confidence: 0.9, grid_relationship: isNeighbor, evidence: [`Grid neighbors: ${isNeighbor}`] }); } } } return gridRelationships; } /** * Build relationship graph */ buildRelationshipGraph(components) { const nodes = components.map(comp => ({ id: comp.id, type: comp.type, position: comp.position, connections: [] })); const edges = []; // This would be populated by all the relationship analysis above // For now, return the structure return { nodes, edges, node_count: nodes.length, edge_count: edges.length, density: edges.length / (nodes.length * (nodes.length - 1) / 2) }; } /** * Analyze interaction flows */ async analyzeInteractionFlows(components, textElements) { const flows = []; // Form submission flows const formFlows = this.analyzeFormFlows(components, textElements); flows.push(...formFlows); // Navigation flows const navFlows = this.analyzeNavigationFlows(components, textElements); flows.push(...navFlows); // Call-to-action flows const ctaFlows = this.analyzeCTAFlows(components, textElements); flows.push(...ctaFlows); return { total_flows: flows.length, by_type: this.groupFlowsByType(flows), flows }; } /** * Analyze form interaction flows */ analyzeFormFlows(components, textElements) { const flows = []; const inputs = components.filter(comp => comp.type === 'input'); const buttons = components.filter(comp => comp.type === 'button'); // Create form submission flows for (const button of buttons) { if (button.text_content && /submit|send|save|register/i.test(button.text_content)) { const relatedInputs = inputs.filter(input => { const distance = this.calculateDistances(input.position, button.position).center_to_center; return distance < 400; }); if (relatedInputs.length > 0) { flows.push({ type: 'form_submission', steps: [ ...relatedInputs.map(input => ({ component: input.id, action: 'input' })), { component: button.id, action: 'submit' } ], confidence: 0.8, description: 'User fills form inputs and submits' }); } } } return flows; } /** * Identify component groups */ async identifyComponentGroups(components) { const groups = []; // Group by proximity and type similarity const proximityGroups = this.groupByProximity(components); groups.push(...proximityGroups); // Group by functional relationships const functionalGroups = this.groupByFunction(components); groups.push(...functionalGroups); // Group by visual similarity const visualGroups = this.groupByVisualSimilarity(components); groups.push(...visualGroups); return { total_groups: groups.length, by_type: this.groupGroupsByType(groups), groups }; } /** * Group components by proximity */ groupByProximity(components) { const groups = []; const used = new Set(); for (const comp of components) { if (used.has(comp.id)) continue; const group = { type: 'proximity', components: [comp.id], center: { x: comp.position.x, y: comp.position.y }, bounds: { ...comp.position } }; used.add(comp.id); // Find nearby components for (const other of components) { if (used.has(other.id)) continue; const distance = this.calculateDistances(comp.position, other.position).center_to_center; if (distance < 150) { group.components.push(other.id); used.add(other.id); // Update group bounds group.bounds = this.expandBounds(group.bounds, other.position); } } if (group.components.length > 1) { groups.push(group); } } return groups; } /** * Generate relationship summary */ generateRelationshipSummary(relationships) { const summary = { total_relationships: 0, by_type: {}, strongest_relationships: [], relationship_density: 0, common_patterns: [] }; // Count relationships by type const allRelationships = [ ...(relationships.spatial_relationships?.relationships || []), ...(relationships.functional_relationships?.relationships || []), ...(relationships.hierarchical_relationships?.relationships || []), ...(relationships.semantic_relationships?.relationships || []), ...(relationships.layout_relationships?.relationships || []) ]; summary.total_relationships = allRelationships.length; // Group by type allRelationships.forEach(rel => { summary.by_type[rel.type] = (summary.by_type[rel.type] || 0) + 1; }); // Find strongest relationships summary.strongest_relationships = allRelationships .filter(rel => (rel.strength || rel.confidence) > 0.8) .sort((a, b) => (b.strength || b.confidence) - (a.strength || a.confidence)) .slice(0, 5); return summary; } // Helper methods calculateDistances(pos1, pos2) { const center1 = { x: pos1.x + pos1.width / 2, y: pos1.y + pos1.height / 2 }; const center2 = { x: pos2.x + pos2.width / 2, y: pos2.y + pos2.height / 2 }; const center_to_center = Math.sqrt( Math.pow(center2.x - center1.x, 2) + Math.pow(center2.y - center1.y, 2) ); // Calculate edge-to-edge distance const edge_to_edge = Math.max(0, center_to_center - (pos1.width + pos2.width) / 2 - (pos1.height + pos2.height) / 2); return { center_to_center, edge_to_edge }; } analyzeAlignment(pos1, pos2) { const horizontalAligned = Math.abs(pos1.y - pos2.y) <= this.options.alignmentTolerance || Math.abs((pos1.y + pos1.height / 2) - (pos2.y + pos2.height / 2)) <= this.options.alignmentTolerance; const verticalAligned = Math.abs(pos1.x - pos2.x) <= this.options.alignmentTolerance || Math.abs((pos1.x + pos1.width / 2) - (pos2.x + pos2.width / 2)) <= this.options.alignmentTolerance; return { horizontal_aligned: horizontalAligned, vertical_aligned: verticalAligned }; } isContained(inner, outer) { return inner.x >= (outer.x - this.options.containmentPadding) && inner.y >= (outer.y - this.options.containmentPadding) && (inner.x + inner.width) <= (outer.x + outer.width + this.options.containmentPadding) && (inner.y + inner.height) <= (outer.y + outer.height + this.options.containmentPadding); } hasOverlap(pos1, pos2) { return !(pos1.x + pos1.width < pos2.x || pos2.x + pos2.width < pos1.x || pos1.y + pos1.height < pos2.y || pos2.y + pos2.height < pos1.y); } getRelativePosition(pos1, pos2) { const center1 = { x: pos1.x + pos1.width / 2, y: pos1.y + pos1.height / 2 }; const center2 = { x: pos2.x + pos2.width / 2, y: pos2.y + pos2.height / 2 }; const dx = center2.x - center1.x; const dy = center2.y - center1.y; if (Math.abs(dx) > Math.abs(dy)) { return dx > 0 ? 'right' : 'left'; } else { return dy > 0 ? 'below' : 'above'; } } identifySpatialPatterns(relationships) { const patterns = {}; relationships.forEach(rel => { patterns[rel.subtype] = (patterns[rel.subtype] || 0) + 1; }); return Object.entries(patterns) .sort(([, a], [, b]) => b - a) .map(([pattern, count]) => ({ pattern, count })); } groupRelationshipsByCategory(relationships) { const grouped = {}; relationships.forEach(rel => { const category = rel.subtype.split('_')[0]; if (!grouped[category]) grouped[category] = []; grouped[category].push(rel); }); return grouped; } groupRelationshipsBySubtype(relationships) { const grouped = {}; relationships.forEach(rel => { if (!grouped[rel.subtype]) grouped[rel.subtype] = []; grouped[rel.subtype].push(rel); }); return grouped; } analyzeTextualRelationship(comp1, comp2, pattern, textElements) { // Placeholder for text analysis return 0.5; } getFunctionalEvidence(comp1, comp2, pattern, textElements) { return [`Components match ${pattern.relationship} pattern`]; } calculateHierarchyLevel(component, allComponents) { let level = 0; let current = component; while (true) { const parent = allComponents.find(comp => comp.id !== current.id && this.isContained(current.position, comp.position) ); if (!parent) break; level++; current = parent; } return level; } analyzeSizeHierarchy(components) { // Placeholder for size-based hierarchy analysis return []; } analyzeZIndexHierarchy(components) { // Placeholder for z-index hierarchy analysis return []; } calculateMaxHierarchyDepth(relationships) { const levels = relationships.map(rel => rel.level || 0); return Math.max(...levels, 0); } analyzeFlexboxRelationships(components, layoutAnalysis) { // Placeholder for flexbox relationship analysis return []; } assignComponentsToGrid(components, gridAnalysis) { // Placeholder for grid assignment return {}; } areGridNeighbors(cell1, cell2) { // Placeholder for grid neighbor detection return false; } analyzeNavigationFlows(components, textElements) { // Placeholder for navigation flow analysis return []; } analyzeCTAFlows(components, textElements) { // Placeholder for CTA flow analysis return []; } groupFlowsByType(flows) { const grouped = {}; flows.forEach(flow => { if (!grouped[flow.type]) grouped[flow.type] = []; grouped[flow.type].push(flow); }); return grouped; } groupByFunction(components) { // Placeholder for functional grouping return []; } groupByVisualSimilarity(components) { // Placeholder for visual similarity grouping return []; } groupGroupsByType(groups) { const grouped = {}; groups.forEach(group => { if (!grouped[group.type]) grouped[group.type] = []; grouped[group.type].push(group); }); return grouped; } expandBounds(bounds, position) { return { x: Math.min(bounds.x, position.x), y: Math.min(bounds.y, position.y), width: Math.max(bounds.x + bounds.width, position.x + position.width) - Math.min(bounds.x, position.x), height: Math.max(bounds.y + bounds.height, position.y + position.height) - Math.min(bounds.y, position.y) }; } } module.exports = ComponentRelationshipMapper;