sf-agent-framework/tools/document-sharder.js

AI Agent Orchestration Framework for Salesforce Development - Two-phase architecture with 70% context reduction

#!/usr/bin/env node

/**
 * Document Sharding System for SF-Agent Framework
 * Breaks large planning documents into story-sized pieces for lean development
 * Based on advanced document sharding principles
 */

const fs = require('fs-extra');
const path = require('path');
const chalk = require('chalk');
const { marked } = require('marked');

class SalesforceDocumentSharder {
  constructor(options = {}) {
    this.options = {
      maxStorySize: options.maxStorySize || 15000, // tokens
      outputDir: options.outputDir || 'docs/sharded',
      storyDir: options.storyDir || 'docs/stories',
      verbose: options.verbose || false,
      ...options,
    };

    this.stats = {
      totalSections: 0,
      storiesCreated: 0,
      epicsCrea: 0,
      documentsProcessed: 0,
    };
  }

  /**
   * Main sharding function
   */
  async shardDocument(documentPath) {
    console.log(chalk.blue(`\n📄 Sharding document: ${documentPath}`));

    try {
      const content = await fs.readFile(documentPath, 'utf-8');
      const docName = path.basename(documentPath, '.md');
      const outputPath = path.join(this.options.outputDir, docName);

      // Create output directory
      await fs.ensureDir(outputPath);

      // Parse document structure
      const structure = this.parseDocumentStructure(content);

      // Shard based on document type
      if (docName.includes('requirements')) {
        await this.shardRequirements(structure, outputPath);
      } else if (docName.includes('architecture')) {
        await this.shardArchitecture(structure, outputPath);
      } else if (docName.includes('technical')) {
        await this.shardTechnicalDesign(structure, outputPath);
      } else {
        await this.shardGeneric(structure, outputPath);
      }

      this.stats.documentsProcessed++;
      this.printStats();
    } catch (error) {
      console.error(chalk.red(`❌ Error sharding document: ${error.message}`));
      throw error;
    }
  }

  /**
   * Parse markdown document into structured sections
   */
  parseDocumentStructure(content) {
    const lines = content.split('\n');
    const structure = {
      title: '',
      sections: [],
      metadata: {},
    };

    let currentSection = null;
    let currentSubsection = null;
    let contentBuffer = [];

    for (const line of lines) {
      // Main title (# Title)
      if (line.startsWith('# ')) {
        structure.title = line.substring(2).trim();
      }
      // Section (## Section)
      else if (line.startsWith('## ')) {
        if (currentSection) {
          if (currentSubsection) {
            currentSubsection.content = contentBuffer.join('\n');
            currentSection.subsections.push(currentSubsection);
            currentSubsection = null;
          } else {
            currentSection.content = contentBuffer.join('\n');
          }
          structure.sections.push(currentSection);
        }

        currentSection = {
          title: line.substring(3).trim(),
          level: 2,
          content: '',
          subsections: [],
        };
        contentBuffer = [];
      }
      // Subsection (### Subsection)
      else if (line.startsWith('### ')) {
        if (currentSubsection) {
          currentSubsection.content = contentBuffer.join('\n');
          currentSection.subsections.push(currentSubsection);
        }

        currentSubsection = {
          title: line.substring(4).trim(),
          level: 3,
          content: '',
        };
        contentBuffer = [];
      }
      // Content
      else {
        contentBuffer.push(line);
      }
    }

    // Save last section
    if (currentSection) {
      if (currentSubsection) {
        currentSubsection.content = contentBuffer.join('\n');
        currentSection.subsections.push(currentSubsection);
      } else {
        currentSection.content = contentBuffer.join('\n');
      }
      structure.sections.push(currentSection);
    }

    return structure;
  }

  /**
   * Shard requirements document into epics and stories
   */
  async shardRequirements(structure, outputPath) {
    console.log(chalk.cyan('  📋 Sharding requirements document...'));

    const epics = [];
    const metadata = {
      source: 'requirements.md',
      sharded_date: new Date().toISOString(),
      total_epics: 0,
      total_stories: 0,
    };

    // Find user stories or functional requirements sections
    for (const section of structure.sections) {
      if (this.isEpicSection(section.title)) {
        const epic = {
          id: `EPIC-${String(epics.length + 1).padStart(3, '0')}`,
          title: section.title,
          description: section.content,
          stories: [],
        };

        // Extract stories from subsections
        for (const subsection of section.subsections) {
          if (this.isStorySection(subsection.title)) {
            const story = {
              id: `${epic.id}-${String(epic.stories.length + 1).padStart(3, '0')}`,
              title: subsection.title,
              content: subsection.content,
              epic_id: epic.id,
              acceptance_criteria: this.extractAcceptanceCriteria(subsection.content),
            };
            epic.stories.push(story);
            metadata.total_stories++;
          }
        }

        epics.push(epic);
        metadata.total_epics++;

        // Write epic file
        await this.writeEpicFile(epic, outputPath);
      }
    }

    // Write metadata
    await this.writeMetadata(metadata, outputPath);

    this.stats.epicsCreated += epics.length;
    console.log(
      chalk.green(`  ✅ Created ${epics.length} epics with ${metadata.total_stories} stories`)
    );
  }

  /**
   * Shard architecture document into component-based sections
   */
  async shardArchitecture(structure, outputPath) {
    console.log(chalk.cyan('  🏗️  Sharding architecture document...'));

    const components = [];
    const metadata = {
      source: 'architecture.md',
      sharded_date: new Date().toISOString(),
      total_components: 0,
    };

    // Architecture-specific sections
    const architectureSections = [
      'System Architecture',
      'Data Model',
      'Integration Architecture',
      'Security Architecture',
      'Technical Components',
      'API Design',
      'Performance Architecture',
    ];

    for (const section of structure.sections) {
      if (architectureSections.some((arch) => section.title.includes(arch))) {
        const component = {
          id: `ARCH-${String(components.length + 1).padStart(3, '0')}`,
          title: section.title,
          content: section.content,
          subsections: section.subsections,
          references: this.extractReferences(section.content),
        };

        components.push(component);
        metadata.total_components++;

        // Write component file
        await this.writeComponentFile(component, outputPath);
      }
    }

    // Write metadata
    await this.writeMetadata(metadata, outputPath);

    console.log(chalk.green(`  ✅ Created ${components.length} architecture components`));
  }

  /**
   * Shard technical design document
   */
  async shardTechnicalDesign(structure, outputPath) {
    console.log(chalk.cyan('  🔧 Sharding technical design document...'));

    const technicalPieces = [];

    for (const section of structure.sections) {
      const piece = {
        id: `TECH-${String(technicalPieces.length + 1).padStart(3, '0')}`,
        title: section.title,
        content: this.enrichTechnicalContent(section),
        implementation_notes: this.extractImplementationNotes(section.content),
      };

      technicalPieces.push(piece);
      await this.writeTechnicalPiece(piece, outputPath);
    }

    console.log(chalk.green(`  ✅ Created ${technicalPieces.length} technical pieces`));
  }

  /**
   * Generic sharding for other document types
   */
  async shardGeneric(structure, outputPath) {
    console.log(chalk.cyan('  📝 Sharding document into sections...'));

    let shardIndex = 0;
    for (const section of structure.sections) {
      const shard = {
        id: `SHARD-${String(++shardIndex).padStart(3, '0')}`,
        title: section.title,
        content: section.content,
        subsections: section.subsections,
      };

      await this.writeShardFile(shard, outputPath);
    }

    console.log(chalk.green(`  ✅ Created ${shardIndex} shards`));
  }

  /**
   * Check if section represents an epic
   */
  isEpicSection(title) {
    const epicIndicators = [
      'Feature',
      'Module',
      'Epic',
      'Capability',
      'User Story',
      'Functional Requirement',
      'Business Requirement',
    ];

    return epicIndicators.some((indicator) =>
      title.toLowerCase().includes(indicator.toLowerCase())
    );
  }

  /**
   * Check if section represents a story
   */
  isStorySection(title) {
    const storyIndicators = ['Story', 'Requirement', 'Scenario', 'Use Case', 'Task'];

    return storyIndicators.some((indicator) =>
      title.toLowerCase().includes(indicator.toLowerCase())
    );
  }

  /**
   * Extract acceptance criteria from content
   */
  extractAcceptanceCriteria(content) {
    const criteria = [];
    const lines = content.split('\n');

    let inCriteriaSection = false;
    for (const line of lines) {
      if (
        line.toLowerCase().includes('acceptance criteria') ||
        line.toLowerCase().includes('success criteria')
      ) {
        inCriteriaSection = true;
        continue;
      }

      if (inCriteriaSection) {
        if (line.startsWith('- ') || line.startsWith('* ') || line.match(/^\d+\./)) {
          criteria.push(line);
        } else if (line.startsWith('#')) {
          inCriteriaSection = false;
        }
      }
    }

    return criteria;
  }

  /**
   * Extract references from content
   */
  extractReferences(content) {
    const references = [];
    const patterns = [
      /\[([^\]]+)\]\(([^)]+)\)/g, // Markdown links
      /https?:\/\/[^\s]+/g, // URLs
      /`([^`]+)`/g, // Code references
    ];

    for (const pattern of patterns) {
      const matches = content.matchAll(pattern);
      for (const match of matches) {
        references.push(match[0]);
      }
    }

    return [...new Set(references)];
  }

  /**
   * Extract implementation notes
   */
  extractImplementationNotes(content) {
    const notes = [];
    const lines = content.split('\n');

    for (const line of lines) {
      if (
        line.toLowerCase().includes('note:') ||
        line.toLowerCase().includes('important:') ||
        line.toLowerCase().includes('todo:')
      ) {
        notes.push(line);
      }
    }

    return notes;
  }

  /**
   * Enrich technical content with additional context
   */
  enrichTechnicalContent(section) {
    let enrichedContent = section.content;

    // Add subsection content
    if (section.subsections.length > 0) {
      enrichedContent += '\n\n### Subsections\n';
      for (const sub of section.subsections) {
        enrichedContent += `\n#### ${sub.title}\n${sub.content}`;
      }
    }

    return enrichedContent;
  }

  /**
   * Write epic file
   */
  async writeEpicFile(epic, outputPath) {
    const filename = `${epic.id}-${this.sanitizeFilename(epic.title)}.md`;
    const filepath = path.join(outputPath, 'epics', filename);

    await fs.ensureDir(path.join(outputPath, 'epics'));

    const content = `# ${epic.id}: ${epic.title}

## Description
${epic.description}

## Stories
${epic.stories.map((story) => `- **${story.id}**: ${story.title}`).join('\n')}

## Total Stories: ${epic.stories.length}

---

### Story Details

${epic.stories
  .map(
    (story) => `
#### ${story.id}: ${story.title}
${story.content}

**Acceptance Criteria:**
${story.acceptance_criteria.join('\n')}
`
  )
  .join('\n')}
`;

    await fs.writeFile(filepath, content);
    this.stats.storiesCreated += epic.stories.length;
  }

  /**
   * Write component file
   */
  async writeComponentFile(component, outputPath) {
    const filename = `${component.id}-${this.sanitizeFilename(component.title)}.md`;
    const filepath = path.join(outputPath, 'components', filename);

    await fs.ensureDir(path.join(outputPath, 'components'));

    const content = `# ${component.id}: ${component.title}

## Content
${component.content}

## Subsections
${component.subsections
  .map(
    (sub) => `
### ${sub.title}
${sub.content}
`
  )
  .join('\n')}

## References
${component.references.map((ref) => `- ${ref}`).join('\n')}
`;

    await fs.writeFile(filepath, content);
  }

  /**
   * Write technical piece file
   */
  async writeTechnicalPiece(piece, outputPath) {
    const filename = `${piece.id}-${this.sanitizeFilename(piece.title)}.md`;
    const filepath = path.join(outputPath, 'technical', filename);

    await fs.ensureDir(path.join(outputPath, 'technical'));

    const content = `# ${piece.id}: ${piece.title}

## Technical Content
${piece.content}

## Implementation Notes
${piece.implementation_notes.join('\n')}
`;

    await fs.writeFile(filepath, content);
  }

  /**
   * Write generic shard file
   */
  async writeShardFile(shard, outputPath) {
    const filename = `${shard.id}-${this.sanitizeFilename(shard.title)}.md`;
    const filepath = path.join(outputPath, filename);

    const content = `# ${shard.id}: ${shard.title}

${shard.content}

${shard.subsections
  .map(
    (sub) => `
## ${sub.title}
${sub.content}
`
  )
  .join('\n')}
`;

    await fs.writeFile(filepath, content);
  }

  /**
   * Write metadata file
   */
  async writeMetadata(metadata, outputPath) {
    const filepath = path.join(outputPath, 'metadata.json');
    await fs.writeJson(filepath, metadata, { spaces: 2 });
  }

  /**
   * Sanitize filename
   */
  sanitizeFilename(name) {
    return name
      .toLowerCase()
      .replace(/[^a-z0-9]/g, '-')
      .replace(/-+/g, '-')
      .replace(/^-|-$/g, '')
      .substring(0, 50);
  }

  /**
   * Print statistics
   */
  printStats() {
    console.log(chalk.yellow('\n📊 Sharding Statistics:'));
    console.log(`  Documents: ${this.stats.documentsProcessed}`);
    console.log(`  Epics: ${this.stats.epicsCreated}`);
    console.log(`  Stories: ${this.stats.storiesCreated}`);
    console.log(`  Sections: ${this.stats.totalSections}`);
  }
}

// CLI Interface
if (require.main === module) {
  const program = require('commander');

  program
    .version('1.0.0')
    .description('Shard Salesforce planning documents for lean development')
    .argument('<document>', 'Path to document to shard')
    .option('-o, --output <dir>', 'Output directory', 'docs/sharded')
    .option('-s, --stories <dir>', 'Stories directory', 'docs/stories')
    .option('-m, --max-size <tokens>', 'Max story size in tokens', '15000')
    .option('-v, --verbose', 'Verbose output')
    .action(async (document, options) => {
      try {
        const sharder = new SalesforceDocumentSharder({
          outputDir: options.output,
          storyDir: options.stories,
          maxStorySize: parseInt(options.maxSize),
          verbose: options.verbose,
        });

        await sharder.shardDocument(document);

        console.log(chalk.green('\n✅ Document sharding complete!'));
        console.log(chalk.blue(`📁 Output: ${options.output}`));
      } catch (error) {
        console.error(chalk.red(`\n❌ Error: ${error.message}`));
        process.exit(1);
      }
    });

  program.parse();
}

module.exports = SalesforceDocumentSharder;