pit-manager
Version:
Centralized prompt management system for Human Behavior AI agents
457 lines (456 loc) • 18.7 kB
JavaScript
;
/**
* Execution tracking for PIT TypeScript - stores execution data for dashboard.
* Matches Python's execution storage format exactly.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExecutionTracker = void 0;
const fs = require("fs/promises");
const path = require("path");
class ExecutionTracker {
constructor(repoPath = '.pit') {
this.repoPath = path.resolve(repoPath);
this.executionsDir = path.join(this.repoPath, 'executions');
this.indexFile = path.join(this.executionsDir, 'index.json');
this.index = this.loadIndexSync();
}
loadIndexSync() {
try {
const data = require('fs').readFileSync(this.indexFile, 'utf8');
return JSON.parse(data);
}
catch {
return {
executions: [],
chains: {},
metrics: {
total_executions: 0,
total_tokens: 0,
total_cost: 0.0,
models_used: {}
}
};
}
}
async loadIndex() {
try {
await fs.access(this.indexFile);
const data = await fs.readFile(this.indexFile, 'utf8');
return JSON.parse(data);
}
catch {
return {
executions: [],
chains: {},
metrics: {
total_executions: 0,
total_tokens: 0,
total_cost: 0.0,
models_used: {}
}
};
}
}
async saveIndex() {
await fs.writeFile(this.indexFile, JSON.stringify(this.index, null, 2));
}
async trackExecution(executionId, model, promptHash, tag, tokens, latencyMs, cost, chainId = null, parentExecutionId = null, metadata = {}) {
// Ensure executions directory exists
await fs.mkdir(this.executionsDir, { recursive: true });
// Reload index to get latest state
this.index = await this.loadIndex();
const execution = {
id: executionId,
timestamp: new Date().toISOString(),
model,
prompt_hash: promptHash,
tag,
tokens,
latency_ms: latencyMs,
cost,
chain_id: chainId,
parent_execution_id: parentExecutionId,
metadata: metadata || {}
};
// Save execution file
const execFile = path.join(this.executionsDir, `${executionId}.json`);
await fs.writeFile(execFile, JSON.stringify(execution, null, 2));
// Update index
this.index.executions.push({
id: executionId,
timestamp: execution.timestamp,
model,
tag,
tokens: tokens.total || 0,
cost,
chain_id: chainId
});
// Update metrics
this.index.metrics.total_executions += 1;
this.index.metrics.total_tokens += tokens.total || 0;
this.index.metrics.total_cost += cost;
// Update model usage
if (!this.index.metrics.models_used[model]) {
this.index.metrics.models_used[model] = {
count: 0,
tokens: 0,
cost: 0.0
};
}
this.index.metrics.models_used[model].count += 1;
this.index.metrics.models_used[model].tokens += tokens.total || 0;
this.index.metrics.models_used[model].cost += cost;
// Update chains
if (chainId) {
if (!this.index.chains[chainId]) {
this.index.chains[chainId] = {
id: chainId,
executions: [],
created: new Date().toISOString()
};
}
this.index.chains[chainId].executions.push(executionId);
}
await this.saveIndex();
// Queue for online sync if in online mode
console.log('[DEBUG] About to call queueForSync');
await this.queueForSync(execution);
console.log('[DEBUG] queueForSync completed');
return executionId;
}
async queueForSync(execution) {
/**
* Queue execution for online sync.
* This enables automatic syncing to Supabase when in online mode.
*/
// Try to load .env file if PIT env vars not already set
if (!process.env.PIT_REPO_KEY || !process.env.PIT_SUPABASE_URL || !process.env.PIT_SUPABASE_SERVICE_ROLE_KEY) {
try {
// Check for .env file in repo root
const envPath = path.join(path.dirname(this.repoPath), '.env');
if (await fs.access(envPath).then(() => true).catch(() => false)) {
const envContent = await fs.readFile(envPath, 'utf-8');
// Load PIT_REPO_KEY
if (!process.env.PIT_REPO_KEY) {
const repoKeyMatch = envContent.match(/PIT_REPO_KEY=(.+)/);
if (repoKeyMatch) {
process.env.PIT_REPO_KEY = repoKeyMatch[1].trim();
}
}
// Load PIT_SUPABASE_URL
if (!process.env.PIT_SUPABASE_URL) {
const urlMatch = envContent.match(/PIT_SUPABASE_URL=(.+)/);
if (urlMatch) {
process.env.PIT_SUPABASE_URL = urlMatch[1].trim();
}
}
// Load PIT_SUPABASE_SERVICE_ROLE_KEY
if (!process.env.PIT_SUPABASE_SERVICE_ROLE_KEY) {
const keyMatch = envContent.match(/PIT_SUPABASE_SERVICE_ROLE_KEY=(.+)/);
if (keyMatch) {
process.env.PIT_SUPABASE_SERVICE_ROLE_KEY = keyMatch[1].trim();
}
}
}
}
catch (error) {
// Silently ignore .env loading errors
}
}
// Check if we have ALL required credentials for online mode
const repoKey = process.env.PIT_REPO_KEY;
const supabaseUrl = process.env.PIT_SUPABASE_URL;
const supabaseKey = process.env.PIT_SUPABASE_SERVICE_ROLE_KEY;
if (!repoKey || !supabaseUrl || !supabaseKey) {
// Not in online mode - missing required credentials
return;
}
if (process.env.PIT_DEBUG) {
console.log('[DEBUG] queueForSync: All credentials present, queuing for sync');
}
// Create sync queue file path
const syncQueueFile = path.join(this.repoPath, 'sync_queue.json');
// Load existing queue
let queue = [];
try {
const queueContent = await fs.readFile(syncQueueFile, 'utf-8');
queue = JSON.parse(queueContent);
}
catch (error) {
// File doesn't exist or is invalid, start with empty queue
queue = [];
}
// Add to queue
queue.push(execution);
// Save queue
try {
await fs.writeFile(syncQueueFile, JSON.stringify(queue, null, 2));
if (process.env.PIT_DEBUG) {
console.log(`[DEBUG] Queued execution ${execution.id} (${queue.length} total in queue)`);
}
}
catch (error) {
if (process.env.PIT_DEBUG) {
console.log('[DEBUG] Error saving sync queue:', error);
}
}
// Try to sync immediately in background
this.trySyncAsync();
}
trySyncAsync() {
/**
* Try to sync executions in background using OnlineSyncManager.
* This runs asynchronously without blocking the main execution.
*/
// Initialize online sync if not already done
try {
const { initializeOnlineSync } = require('./online-sync');
const syncManager = initializeOnlineSync(this.repoPath);
// Trigger immediate sync attempt
syncManager.syncExecutions().catch(() => {
// Ignore sync errors - they're logged internally
});
}
catch (error) {
// Online sync module not available or failed to initialize
if (process.env.PIT_DEBUG) {
console.log('[DEBUG] Online sync not available:', error);
}
}
}
async getExecutions(limit) {
const executions = [];
// Sort by timestamp descending
const sortedExecs = [...this.index.executions].sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
const execsToLoad = limit ? sortedExecs.slice(0, limit) : sortedExecs;
// Load full execution data
for (const execSummary of execsToLoad) {
const execFile = path.join(this.executionsDir, `${execSummary.id}.json`);
try {
const data = await fs.readFile(execFile, 'utf8');
executions.push(JSON.parse(data));
}
catch {
// Skip if file doesn't exist
}
}
return executions;
}
getMetrics() {
return this.index.metrics;
}
getChains() {
return this.index.chains;
}
get executionsDirectory() {
return this.executionsDir;
}
async getExecutionById(executionId) {
try {
const execFile = path.join(this.executionsDir, `${executionId}.json`);
const data = await fs.readFile(execFile, 'utf8');
return JSON.parse(data);
}
catch {
return null;
}
}
async getTimeline(_days = 7) {
const timeline = [];
const dailyStats = {};
for (const execSummary of this.index.executions) {
const timestamp = new Date(execSummary.timestamp);
const dateKey = timestamp.toISOString().split('T')[0];
if (!dailyStats[dateKey]) {
dailyStats[dateKey] = {
date: dateKey,
executions: 0,
tokens: 0,
cost: 0.0,
models: new Set()
};
}
dailyStats[dateKey].executions += 1;
dailyStats[dateKey].tokens += execSummary.tokens || 0;
dailyStats[dateKey].cost += execSummary.cost || 0.0;
dailyStats[dateKey].models.add(execSummary.model || 'unknown');
}
// Convert to array
for (const [, stats] of Object.entries(dailyStats)) {
timeline.push({
date: stats.date,
executions: stats.executions,
tokens: stats.tokens,
cost: stats.cost,
models: Array.from(stats.models)
});
}
// Sort by date
timeline.sort((a, b) => a.date.localeCompare(b.date));
return timeline;
}
/**
* Get aggregated chain execution summary
*/
async getChainSummary(chainId) {
try {
// Load all executions for this chain
const chainExecutions = [];
const executionFiles = await fs.readdir(this.executionsDir);
for (const file of executionFiles) {
if (file.endsWith('.json') && file !== 'index.json') {
try {
const execData = await fs.readFile(path.join(this.executionsDir, file), 'utf8');
const execution = JSON.parse(execData);
if (execution.chain_id === chainId) {
chainExecutions.push(execution);
}
}
catch {
// Skip invalid files
}
}
}
if (chainExecutions.length === 0) {
return null;
}
// Build parent-child relationships
const parentChildMap = new Map();
const rootExecutions = [];
for (const exec of chainExecutions) {
if (exec.parent_execution_id) {
const siblings = parentChildMap.get(exec.parent_execution_id) || [];
siblings.push(exec.id);
parentChildMap.set(exec.parent_execution_id, siblings);
}
else {
rootExecutions.push(exec);
}
}
// Group executions by tag
const tagGroups = new Map();
for (const exec of chainExecutions) {
const tag = exec.tag || 'unknown';
const group = tagGroups.get(tag) || [];
group.push(exec);
tagGroups.set(tag, group);
}
// Calculate agent summaries
const agentSummaries = [];
const processedTags = new Set();
// Process in hierarchical order starting from roots
const processTag = (tag, parentTag) => {
if (processedTags.has(tag))
return;
processedTags.add(tag);
const executions = tagGroups.get(tag) || [];
if (executions.length === 0)
return;
// Calculate statistics
const totalCalls = executions.length;
const avgCalls = totalCalls / rootExecutions.length; // Avg calls per chain execution
let totalPromptTokens = 0;
let totalCompletionTokens = 0;
let totalTokens = 0;
let totalLatency = 0;
for (const exec of executions) {
totalPromptTokens += exec.tokens.prompt || 0;
totalCompletionTokens += exec.tokens.completion || 0;
totalTokens += exec.tokens.total || 0;
totalLatency += exec.latency_ms || 0;
}
// Find child tags
const childTags = new Set();
for (const exec of executions) {
const children = parentChildMap.get(exec.id) || [];
for (const childId of children) {
const childExec = chainExecutions.find(e => e.id === childId);
if (childExec && childExec.tag) {
childTags.add(childExec.tag);
}
}
}
const summary = {
tag,
model: executions[0].model,
provider: executions[0].metadata?.provider || 'unknown',
avgCalls,
totalCalls,
avgTokens: {
prompt: totalPromptTokens / totalCalls,
completion: totalCompletionTokens / totalCalls,
total: totalTokens / totalCalls
},
totalTokens: {
prompt: totalPromptTokens,
completion: totalCompletionTokens,
total: totalTokens
},
avgLatencyMs: totalLatency / totalCalls,
totalLatencyMs: totalLatency,
executionIds: executions.map(e => e.id),
parentTag,
childTags: Array.from(childTags)
};
agentSummaries.push(summary);
// Process child tags
for (const childTag of childTags) {
processTag(childTag, tag);
}
};
// Start processing from root tags
for (const rootExec of rootExecutions) {
if (rootExec.tag) {
processTag(rootExec.tag);
}
}
// Calculate chain-level statistics
const timestamps = chainExecutions.map(e => new Date(e.timestamp).getTime());
const executionDurations = rootExecutions.map(root => {
// Find all executions in this chain instance
const instanceExecs = [root];
const findChildren = (parentId) => {
const children = parentChildMap.get(parentId) || [];
for (const childId of children) {
const child = chainExecutions.find(e => e.id === childId);
if (child) {
instanceExecs.push(child);
findChildren(childId);
}
}
};
findChildren(root.id);
const instanceTimestamps = instanceExecs.map(e => new Date(e.timestamp).getTime());
const latencies = instanceExecs.map(e => e.latency_ms || 0);
return Math.max(...instanceTimestamps) - Math.min(...instanceTimestamps) + Math.max(...latencies);
});
const chainSummary = {
chainId,
chainExecutions: rootExecutions.length,
agentSummaries,
totalTokens: {
prompt: agentSummaries.reduce((sum, a) => sum + a.totalTokens.prompt, 0),
completion: agentSummaries.reduce((sum, a) => sum + a.totalTokens.completion, 0),
total: agentSummaries.reduce((sum, a) => sum + a.totalTokens.total, 0)
},
totalLatencyMs: agentSummaries.reduce((sum, a) => sum + a.totalLatencyMs, 0),
avgLatencyMs: agentSummaries.reduce((sum, a) => sum + a.totalLatencyMs, 0) / rootExecutions.length,
totalCost: chainExecutions.reduce((sum, e) => sum + (e.cost || 0), 0),
avgCost: chainExecutions.reduce((sum, e) => sum + (e.cost || 0), 0) / rootExecutions.length,
firstExecutionTime: new Date(Math.min(...timestamps)).toISOString(),
lastExecutionTime: new Date(Math.max(...timestamps)).toISOString(),
executionTimeRange: {
min: Math.min(...executionDurations),
max: Math.max(...executionDurations),
avg: executionDurations.reduce((a, b) => a + b, 0) / executionDurations.length
}
};
return chainSummary;
}
catch (error) {
console.error('Error getting chain summary:', error);
return null;
}
}
}
exports.ExecutionTracker = ExecutionTracker;