remcode
Version:
Turn your AI assistant into a codebase expert. Intelligent code analysis, semantic search, and software engineering guidance through MCP integration.
254 lines (253 loc) • 12.4 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.processCommand = processCommand;
const chalk_1 = __importDefault(require("chalk"));
const ora_1 = __importDefault(require("ora"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const pipeline_1 = require("../processing/pipeline");
const state_manager_1 = require("../processing/state-manager");
const logger_1 = require("../utils/logger");
const logger = (0, logger_1.getLogger)('ProcessCommand');
function processCommand(program) {
program
.command('process')
.description('Process repository for vectorization (used by GitHub Actions)')
.option('-t, --type <type>', 'Processing type: auto, full, or incremental', 'auto')
.option('-c, --config <path>', 'Path to .remcode config file')
.option('-f, --force', 'Force full reprocessing even if incremental is possible')
.option('-d, --dry-run', 'Show what would be processed without actually processing')
.option('--timeout <seconds>', 'Maximum processing time in seconds', '3600')
.option('-v, --verbose', 'Enable verbose output')
.option('-r, --report <path>', 'Path to save processing report', './processing-report.json')
.action(async (options) => {
const spinner = (0, ora_1.default)('Initializing processing pipeline...').start();
try {
// Get current working directory (should be repository root in GitHub Actions)
const repoPath = process.cwd();
// Validate repository structure
if (!fs.existsSync(path.join(repoPath, '.git'))) {
throw new Error('Not a Git repository. Processing must run in a Git repository.');
}
// Load configuration
const configPath = options.config || path.join(repoPath, '.remcode');
let config = {};
if (fs.existsSync(configPath)) {
config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
logger.info(`Loaded configuration from ${configPath}`);
}
else {
logger.warn('No .remcode config found, using environment variables and defaults');
}
// Get required environment variables
const processingOptions = await buildProcessingOptions(config, options);
if (options.dryRun) {
spinner.stop();
await showDryRun(repoPath, processingOptions, options);
return;
}
// Initialize state manager
const stateManager = new state_manager_1.StateManager(repoPath);
// Determine processing type
const processingType = await determineProcessingType(options.type, stateManager, options.force);
spinner.text = `Starting ${processingType} processing...`;
logger.info(`Processing type determined: ${processingType}`);
// Initialize processing pipeline
const pipeline = new pipeline_1.ProcessingPipeline(repoPath, processingOptions);
const startTime = Date.now();
let result;
// Execute processing based on type
if (processingType === 'full') {
spinner.text = 'Processing all files in repository...';
result = await pipeline.processAll();
}
else {
spinner.text = 'Processing changed files incrementally...';
result = await pipeline.processIncremental();
}
const endTime = Date.now();
const duration = Math.round((endTime - startTime) / 1000);
spinner.stop();
// Display results
console.log(chalk_1.default.green('✅ Processing completed successfully!'));
console.log(chalk_1.default.blue(`📊 Processing Type: ${processingType}`));
console.log(chalk_1.default.blue(`📁 Total Files: ${result.totalFiles}`));
console.log(chalk_1.default.blue(`➕ Added Files: ${result.addedFiles}`));
console.log(chalk_1.default.blue(`✏️ Modified Files: ${result.modifiedFiles}`));
console.log(chalk_1.default.blue(`🗑️ Deleted Files: ${result.deletedFiles}`));
console.log(chalk_1.default.blue(`🧩 Total Chunks: ${result.totalChunks}`));
console.log(chalk_1.default.blue(`🔢 Total Embeddings: ${result.totalEmbeddings}`));
console.log(chalk_1.default.blue(`⏱️ Duration: ${duration}s`));
if (result.errorCount > 0) {
console.log(chalk_1.default.yellow(`⚠️ Errors: ${result.errorCount}`));
}
// Generate processing report
await generateProcessingReport(result, options.reportPath, {
type: processingType,
repoPath,
config: processingOptions,
duration,
timestamp: new Date().toISOString()
});
// Update state with successful processing
await updateProcessingState(stateManager, result);
// Set exit code for GitHub Actions
process.exit(result.errorCount > 0 ? 1 : 0);
}
catch (error) {
spinner.stop();
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(chalk_1.default.red(`❌ Processing failed: ${errorMessage}`));
logger.error(`Processing error: ${errorMessage}`, error);
// Generate error report
await generateErrorReport(error, options.reportPath, {
repoPath: process.cwd(),
timestamp: new Date().toISOString()
});
process.exit(1);
}
});
}
async function buildProcessingOptions(config, options) {
// Get API keys from environment variables (required in GitHub Actions)
const pineconeApiKey = process.env.PINECONE_API_KEY;
const huggingfaceToken = process.env.HUGGINGFACE_TOKEN;
if (!pineconeApiKey) {
throw new Error('PINECONE_API_KEY environment variable is required');
}
if (!huggingfaceToken) {
throw new Error('HUGGINGFACE_TOKEN environment variable is required');
}
// Build options with precedence: config file → env vars → defaults
const processingOptions = {
repoPath: process.cwd(),
pineconeApiKey,
pineconeEnvironment: process.env.PINECONE_ENVIRONMENT || config.vectorization?.pineconeEnvironment || 'gcp-starter',
pineconeIndexName: process.env.PINECONE_INDEX_NAME || config.vectorization?.indexName ||
`remcode-${path.basename(process.cwd()).replace(/[^a-zA-Z0-9]/g, '-').toLowerCase()}`,
pineconeNamespace: process.env.PINECONE_NAMESPACE || config.vectorization?.namespace || 'main',
embeddingModel: process.env.EMBEDDING_MODEL || config.vectorization?.embeddingModel || 'microsoft/graphcodebert-base',
batchSize: parseInt(process.env.BATCH_SIZE || config.vectorization?.batchSize || '10'),
dryRun: options.dryRun || false,
includeTests: process.env.INCLUDE_TESTS === 'true' || config.advanced?.includeTests !== false
};
logger.info('Processing options configured', {
indexName: processingOptions.pineconeIndexName,
namespace: processingOptions.pineconeNamespace,
model: processingOptions.embeddingModel,
batchSize: processingOptions.batchSize
});
return processingOptions;
}
async function determineProcessingType(requestedType, stateManager, force) {
if (force) {
return 'full';
}
if (requestedType === 'full') {
return 'full';
}
if (requestedType === 'incremental') {
return 'incremental';
}
// Auto-detection logic
const stateExists = await stateManager.exists();
if (!stateExists) {
logger.info('No state file found, defaulting to full processing');
return 'full';
}
const state = await stateManager.loadState();
if (!state || !state.processing?.lastCommit) {
logger.info('No previous commit found in state, defaulting to full processing');
return 'full';
}
logger.info('State file found with previous commit, using incremental processing');
return 'incremental';
}
async function showDryRun(repoPath, options, cmdOptions) {
console.log(chalk_1.default.cyan('🔍 DRY RUN - Processing Preview'));
console.log(chalk_1.default.blue(`📁 Repository: ${repoPath}`));
console.log(chalk_1.default.blue(`🔧 Processing Type: ${cmdOptions.type}`));
console.log(chalk_1.default.blue(`📊 Index Name: ${options.pineconeIndexName}`));
console.log(chalk_1.default.blue(`🏷️ Namespace: ${options.pineconeNamespace}`));
console.log(chalk_1.default.blue(`🤖 Embedding Model: ${options.embeddingModel}`));
console.log(chalk_1.default.blue(`📦 Batch Size: ${options.batchSize}`));
console.log(chalk_1.default.blue(`🧪 Include Tests: ${options.includeTests}`));
console.log(chalk_1.default.cyan('\n✅ Configuration is valid. Run without --dry-run to execute processing.'));
}
async function generateProcessingReport(result, reportPath, metadata) {
const report = {
success: true,
metadata,
results: result,
generatedAt: new Date().toISOString()
};
const finalReportPath = reportPath || './processing-report.json';
await fs.promises.writeFile(finalReportPath, JSON.stringify(report, null, 2), 'utf8');
console.log(chalk_1.default.blue(`📄 Processing report saved to: ${finalReportPath}`));
}
async function generateErrorReport(error, reportPath, metadata) {
const report = {
success: false,
error: {
message: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
},
metadata,
generatedAt: new Date().toISOString()
};
const finalReportPath = reportPath || './processing-error-report.json';
await fs.promises.writeFile(finalReportPath, JSON.stringify(report, null, 2), 'utf8');
console.error(chalk_1.default.red(`📄 Error report saved to: ${finalReportPath}`));
}
async function updateProcessingState(stateManager, result) {
try {
// TODO: Fix updateStatistics call
// const stats: Record<string, any> = {
// filesProcessed: result.totalFiles || 0,
// chunksCreated: result.totalChunks || 0,
// vectorsStored: result.totalEmbeddings || 0,
// lastProcessingDuration: result.durationMs || 0
// };
// await stateManager.updateStatistics(stats as any);
logger.info('Processing state update temporarily disabled');
}
catch (error) {
logger.warn('Failed to update processing state', error);
}
}