pit-manager
Version:
Centralized prompt management system for Human Behavior AI agents
672 lines • 26.9 kB
JavaScript
"use strict";
/**
* Execution tracking for PIT TypeScript - stores execution data for dashboard.
* Matches Python's execution storage format exactly.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExecutionTracker = void 0;
const fs = __importStar(require("fs/promises"));
const path = __importStar(require("path"));
const crypto = __importStar(require("crypto"));
class ExecutionTracker {
repoPath;
executionsDir;
indexFile;
index;
chainGroupsFile;
chainGroups;
activeChainGroupId = undefined;
constructor(repoPath = '.pit') {
this.repoPath = path.resolve(repoPath);
this.executionsDir = path.join(this.repoPath, 'executions');
this.indexFile = path.join(this.executionsDir, 'index.json');
this.chainGroupsFile = path.join(this.executionsDir, 'chain_groups.json');
this.index = this.loadIndexSync();
this.chainGroups = this.loadChainGroupsSync();
}
loadIndexSync() {
try {
const data = require('fs').readFileSync(this.indexFile, 'utf8');
return JSON.parse(data);
}
catch {
return {
executions: [],
chains: {},
metrics: {
total_executions: 0,
total_tokens: 0,
total_cost: 0.0,
models_used: {}
}
};
}
}
loadChainGroupsSync() {
try {
const data = require('fs').readFileSync(this.chainGroupsFile, 'utf8');
return JSON.parse(data);
}
catch {
return {
groups: {},
active_groups: {}
};
}
}
// Removed unused loadIndex method
// Removed unused loadChainGroups method
async saveIndex() {
await fs.mkdir(this.executionsDir, { recursive: true });
await fs.writeFile(this.indexFile, JSON.stringify(this.index, null, 2));
}
async saveChainGroups() {
await fs.mkdir(this.executionsDir, { recursive: true });
await fs.writeFile(this.chainGroupsFile, JSON.stringify(this.chainGroups, null, 2));
}
/**
* Sync a chain execution group to Supabase immediately.
* This ensures the group exists before any executions reference it.
*/
async syncChainGroupToSupabase(groupData) {
try {
// Check if online sync is available
const url = process.env.PIT_SUPABASE_URL;
const serviceKey = process.env.PIT_SUPABASE_SERVICE_ROLE_KEY;
const repoKey = process.env.PIT_REPO_KEY;
if (!url || !serviceKey || !repoKey) {
// Online sync not available, skip
return;
}
// Import Supabase client dynamically to avoid circular dependencies
const { createClient } = await Promise.resolve().then(() => __importStar(require('@supabase/supabase-js')));
const supabaseClient = createClient(url, serviceKey);
// Validate repo key to get proper repo_id
const keyHash = crypto.createHash('sha256').update(repoKey).digest('hex');
const { data: keyData, error: keyError } = await supabaseClient
.from('repo_keys')
.select('repo_id')
.eq('key_hash', keyHash)
.single();
if (keyError || !keyData) {
console.error('[PIT-SYNC] Invalid repository key, cannot sync chain group');
return;
}
const repoId = keyData.repo_id;
// Prepare data for Supabase
const supabaseData = {
id: groupData.id,
repo_id: repoId,
group_id: groupData.group_id,
chain_id: groupData.chain_id,
session_id: groupData.session_id,
process_id: groupData.process_id,
start_time: groupData.start_time,
end_time: groupData.end_time,
status: groupData.status,
total_executions: groupData.total_executions,
total_tokens: groupData.total_tokens,
metadata: groupData.metadata || {},
created_at: groupData.created_at,
updated_at: groupData.updated_at
};
// Insert into chain_execution_groups table
const { error } = await supabaseClient
.from('chain_execution_groups')
.insert(supabaseData);
if (error) {
console.error('[PIT-SYNC] Failed to sync chain group to Supabase:', error.message);
}
else {
console.log('[PIT-SYNC] Chain execution group synced to Supabase:', groupData.id);
}
}
catch (error) {
console.error('[PIT-SYNC] Error syncing chain group to Supabase:', error);
}
}
/**
* Update a chain execution group status in Supabase.
*/
async updateChainGroupInSupabase(groupId, status) {
try {
// Check if online sync is available
const url = process.env.PIT_SUPABASE_URL;
const serviceKey = process.env.PIT_SUPABASE_SERVICE_ROLE_KEY;
if (!url || !serviceKey) {
// Online sync not available, skip
return;
}
const groupData = this.chainGroups.groups[groupId];
if (!groupData) {
return;
}
// Import Supabase client dynamically to avoid circular dependencies
const { createClient } = await Promise.resolve().then(() => __importStar(require('@supabase/supabase-js')));
const supabaseClient = createClient(url, serviceKey);
// Update status and end_time
const { error } = await supabaseClient
.from('chain_execution_groups')
.update({
status: status,
end_time: groupData.end_time,
total_executions: groupData.total_executions,
total_tokens: groupData.total_tokens,
updated_at: groupData.updated_at
})
.eq('id', groupId);
if (error) {
console.error('[PIT-SYNC] Failed to update chain group in Supabase:', error.message);
}
else {
console.log('[PIT-SYNC] Chain execution group updated in Supabase:', groupId, status);
}
}
catch (error) {
console.error('[PIT-SYNC] Error updating chain group in Supabase:', error);
}
}
/**
* Start a new chain execution group for process-unique tracking.
*/
startChainExecutionGroup(chainId, sessionId, processId, metadata) {
// Generate UUID for group ID (required by Supabase)
const groupId = crypto.randomUUID();
// Generate descriptive group ID for display
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const uniqueSuffix = crypto.randomBytes(4).toString('hex');
let descriptiveGroupId;
if (sessionId) {
descriptiveGroupId = `${chainId}-${sessionId}-${timestamp}-${uniqueSuffix}`;
}
else if (processId) {
descriptiveGroupId = `${chainId}-process-${processId}-${timestamp}-${uniqueSuffix}`;
}
else {
descriptiveGroupId = `${chainId}-${timestamp}-${uniqueSuffix}`;
}
// Create group data
const groupData = {
id: groupId,
group_id: descriptiveGroupId, // Descriptive ID for display
chain_id: chainId,
session_id: sessionId,
process_id: processId,
start_time: new Date().toISOString(),
status: 'running',
total_executions: 0,
total_tokens: 0,
metadata: metadata || {},
created_at: new Date().toISOString(),
updated_at: new Date().toISOString()
};
// Store group
this.chainGroups.groups[groupId] = groupData;
// Mark as active for current process
if (processId) {
this.chainGroups.active_groups[processId] = groupId;
}
// Set as active for this tracker instance
this.activeChainGroupId = groupId;
// Save chain groups locally
this.saveChainGroups().catch(console.error);
// Immediately sync to Supabase if online sync is available
this.syncChainGroupToSupabase(groupData).catch(console.error);
return groupId;
}
/**
* Get the active chain group ID for the current process.
*/
getActiveChainGroupId() {
return this.activeChainGroupId;
}
/**
* End a chain execution group.
*/
endChainExecutionGroup(groupId, status = 'completed') {
if (this.chainGroups.groups[groupId]) {
this.chainGroups.groups[groupId].status = status;
this.chainGroups.groups[groupId].end_time = new Date().toISOString();
this.chainGroups.groups[groupId].updated_at = new Date().toISOString();
// Remove from active groups
const processId = this.chainGroups.groups[groupId].process_id;
if (processId && this.chainGroups.active_groups[processId]) {
delete this.chainGroups.active_groups[processId];
}
// Clear active group if it's the current one
if (this.activeChainGroupId === groupId) {
this.activeChainGroupId = undefined;
}
// Save chain groups locally
this.saveChainGroups().catch(console.error);
// Update status in Supabase
this.updateChainGroupInSupabase(groupId, status).catch(console.error);
}
}
/**
* Get a chain execution group by ID.
*/
getChainExecutionGroup(groupId) {
return this.chainGroups.groups[groupId];
}
/**
* Get chain execution groups, optionally filtered by chain ID.
*/
getChainExecutionGroups(chainId) {
let groups = Object.values(this.chainGroups.groups);
if (chainId) {
groups = groups.filter(g => g.chain_id === chainId);
}
return groups.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
}
/**
* Update chain group statistics.
*/
updateChainGroupStats(chainGroupId, tokens) {
if (this.chainGroups.groups[chainGroupId]) {
this.chainGroups.groups[chainGroupId].total_executions += 1;
this.chainGroups.groups[chainGroupId].total_tokens += tokens;
this.chainGroups.groups[chainGroupId].updated_at = new Date().toISOString();
this.saveChainGroups().catch(console.error);
}
}
/**
* Track a model execution with optional chain group association.
*/
async trackExecution(executionId, model, promptHash, tag, tokens, latencyMs, cost, chainId, parentExecutionId, metadata, chainGroupId) {
// Use active chain group if none provided
if (!chainGroupId) {
chainGroupId = this.activeChainGroupId;
}
const execution = {
id: executionId,
timestamp: new Date().toISOString(),
model,
prompt_hash: promptHash,
tag,
tokens,
latency_ms: latencyMs,
cost,
chain_id: chainId || null,
parent_execution_id: parentExecutionId || null,
chain_group_id: chainGroupId, // NEW: Associate with chain group
metadata: metadata || {}
};
// Save execution file
const execFile = path.join(this.executionsDir, `${executionId}.json`);
await fs.mkdir(this.executionsDir, { recursive: true });
await fs.writeFile(execFile, JSON.stringify(execution, null, 2));
// Update index
this.index.executions.push({
id: executionId,
timestamp: execution.timestamp,
model,
tag,
tokens: tokens.total,
cost,
chain_id: chainId || null,
chain_group_id: chainGroupId // NEW: Include in index
});
// Update metrics
this.index.metrics.total_executions += 1;
this.index.metrics.total_tokens += tokens.total;
this.index.metrics.total_cost += cost;
// Update model usage
if (!this.index.metrics.models_used[model]) {
this.index.metrics.models_used[model] = {
count: 0,
tokens: 0,
cost: 0
};
}
this.index.metrics.models_used[model].count += 1;
this.index.metrics.models_used[model].tokens += tokens.total;
this.index.metrics.models_used[model].cost += cost;
// Update chains
if (chainId) {
if (!this.index.chains[chainId]) {
this.index.chains[chainId] = {
id: chainId,
executions: [],
created: new Date().toISOString()
};
}
this.index.chains[chainId].executions.push(executionId);
}
// Update chain group statistics if provided
if (chainGroupId) {
this.updateChainGroupStats(chainGroupId, tokens.total);
}
await this.saveIndex();
// Queue for online sync
this.queueForSync(execution);
return executionId;
}
/**
* Get recent executions, optionally filtered by chain group.
*/
async getExecutions(limit, chainGroupId) {
let executions = [...this.index.executions];
// Filter by chain group if specified
if (chainGroupId) {
executions = executions.filter(e => e.chain_group_id === chainGroupId);
}
// Sort by timestamp descending
executions.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
if (limit) {
executions = executions.slice(0, limit);
}
// Load full execution data
const fullExecutions = [];
for (const execSummary of executions) {
const execFile = path.join(this.executionsDir, `${execSummary.id}.json`);
try {
const data = await fs.readFile(execFile, 'utf8');
fullExecutions.push(JSON.parse(data));
}
catch {
// Skip if file doesn't exist or can't be read
}
}
return fullExecutions;
}
/**
* Get all executions for a specific chain group.
*/
async getExecutionsByChainGroup(chainGroupId) {
return this.getExecutions(undefined, chainGroupId);
}
/**
* Get a specific execution by ID.
*/
async getExecution(executionId) {
const execFile = path.join(this.executionsDir, `${executionId}.json`);
try {
const data = await fs.readFile(execFile, 'utf8');
return JSON.parse(data);
}
catch {
return null;
}
}
/**
* Get execution metrics.
*/
getMetrics() {
return this.index.metrics;
}
/**
* Get chain information.
*/
getChains() {
return this.index.chains;
}
/**
* Get execution timeline for visualization.
*/
getTimeline(_days = 7) {
const timeline = {};
for (const exec of this.index.executions) {
const timestamp = new Date(exec.timestamp);
const dateKey = timestamp.toISOString().split('T')[0];
if (!timeline[dateKey]) {
timeline[dateKey] = {
date: dateKey,
executions: 0,
tokens: 0,
cost: 0,
models: new Set()
};
}
timeline[dateKey].executions += 1;
timeline[dateKey].tokens += exec.tokens;
timeline[dateKey].cost += exec.cost;
timeline[dateKey].models.add(exec.model);
}
return Object.values(timeline)
.map(item => ({
...item,
models: Array.from(item.models)
}))
.sort((a, b) => a.date.localeCompare(b.date));
}
/**
* Queue execution for online sync.
*/
queueForSync(execution) {
const syncQueueFile = path.join(this.repoPath, 'sync_queue.json');
// Load existing queue
let queue = [];
try {
const data = require('fs').readFileSync(syncQueueFile, 'utf8');
queue = JSON.parse(data);
}
catch {
queue = [];
}
// Add to queue
queue.push(execution);
// Save queue
require('fs').writeFileSync(syncQueueFile, JSON.stringify(queue, null, 2));
// Try to sync in background
this.trySyncAsync();
}
/**
* Try to sync executions in background.
*/
trySyncAsync() {
// Initialize online sync if not already done
try {
const { initializeOnlineSync } = require('./online-sync');
const syncManager = initializeOnlineSync(this.repoPath);
// Trigger immediate sync attempt
syncManager.syncExecutions().catch(() => {
// Ignore sync errors - they're logged internally
});
}
catch (error) {
// Online sync module not available or failed to initialize
if (process.env.PIT_DEBUG) {
console.log('[DEBUG] Online sync not available:', error);
}
}
}
/**
* Get aggregated chain execution summary
*/
async getChainSummary(chainId) {
try {
// Load all executions for this chain
const chainExecutions = [];
const executionFiles = await fs.readdir(this.executionsDir);
for (const file of executionFiles) {
if (file.endsWith('.json') && file !== 'index.json') {
try {
const execData = await fs.readFile(path.join(this.executionsDir, file), 'utf8');
const execution = JSON.parse(execData);
if (execution.chain_id === chainId) {
chainExecutions.push(execution);
}
}
catch {
// Skip invalid files
}
}
}
if (chainExecutions.length === 0) {
return null;
}
// Build parent-child relationships
const parentChildMap = new Map();
const rootExecutions = [];
for (const exec of chainExecutions) {
if (exec.parent_execution_id) {
const siblings = parentChildMap.get(exec.parent_execution_id) || [];
siblings.push(exec.id);
parentChildMap.set(exec.parent_execution_id, siblings);
}
else {
rootExecutions.push(exec);
}
}
// Group executions by tag
const tagGroups = new Map();
for (const exec of chainExecutions) {
const tag = exec.tag || 'unknown';
const group = tagGroups.get(tag) || [];
group.push(exec);
tagGroups.set(tag, group);
}
// Calculate agent summaries
const agentSummaries = [];
const processedTags = new Set();
// Process in hierarchical order starting from roots
const processTag = (tag, parentTag) => {
if (processedTags.has(tag))
return;
processedTags.add(tag);
const executions = tagGroups.get(tag) || [];
if (executions.length === 0)
return;
// Calculate statistics
const totalCalls = executions.length;
const avgCalls = totalCalls / rootExecutions.length; // Avg calls per chain execution
let totalPromptTokens = 0;
let totalCompletionTokens = 0;
let totalTokens = 0;
let totalLatency = 0;
for (const exec of executions) {
totalPromptTokens += exec.tokens.prompt || 0;
totalCompletionTokens += exec.tokens.completion || 0;
totalTokens += exec.tokens.total || 0;
totalLatency += exec.latency_ms || 0;
}
// Find child tags
const childTags = new Set();
for (const exec of executions) {
const children = parentChildMap.get(exec.id) || [];
for (const childId of children) {
const childExec = chainExecutions.find(e => e.id === childId);
if (childExec && childExec.tag) {
childTags.add(childExec.tag);
}
}
}
const summary = {
tag,
model: executions[0].model,
provider: executions[0].metadata?.provider || 'unknown',
avgCalls,
totalCalls,
avgTokens: {
prompt: totalPromptTokens / totalCalls,
completion: totalCompletionTokens / totalCalls,
total: totalTokens / totalCalls
},
totalTokens: {
prompt: totalPromptTokens,
completion: totalCompletionTokens,
total: totalTokens
},
avgLatencyMs: totalLatency / totalCalls,
totalLatencyMs: totalLatency,
executionIds: executions.map(e => e.id),
parentTag,
childTags: Array.from(childTags)
};
agentSummaries.push(summary);
// Process child tags
for (const childTag of childTags) {
processTag(childTag, tag);
}
};
// Start processing from root tags
for (const rootExec of rootExecutions) {
if (rootExec.tag) {
processTag(rootExec.tag);
}
}
// Calculate chain-level statistics
const timestamps = chainExecutions.map(e => new Date(e.timestamp).getTime());
const executionDurations = rootExecutions.map(root => {
// Find all executions in this chain instance
const instanceExecs = [root];
const findChildren = (parentId) => {
const children = parentChildMap.get(parentId) || [];
for (const childId of children) {
const child = chainExecutions.find(e => e.id === childId);
if (child) {
instanceExecs.push(child);
findChildren(childId);
}
}
};
findChildren(root.id);
const instanceTimestamps = instanceExecs.map(e => new Date(e.timestamp).getTime());
const latencies = instanceExecs.map(e => e.latency_ms || 0);
return Math.max(...instanceTimestamps) - Math.min(...instanceTimestamps) + Math.max(...latencies);
});
const chainSummary = {
chainId,
chainExecutions: rootExecutions.length,
agentSummaries,
totalTokens: {
prompt: agentSummaries.reduce((sum, a) => sum + a.totalTokens.prompt, 0),
completion: agentSummaries.reduce((sum, a) => sum + a.totalTokens.completion, 0),
total: agentSummaries.reduce((sum, a) => sum + a.totalTokens.total, 0)
},
totalLatencyMs: agentSummaries.reduce((sum, a) => sum + a.totalLatencyMs, 0),
avgLatencyMs: agentSummaries.reduce((sum, a) => sum + a.totalLatencyMs, 0) / rootExecutions.length,
totalCost: chainExecutions.reduce((sum, e) => sum + (e.cost || 0), 0),
avgCost: chainExecutions.reduce((sum, e) => sum + (e.cost || 0), 0) / rootExecutions.length,
firstExecutionTime: new Date(Math.min(...timestamps)).toISOString(),
lastExecutionTime: new Date(Math.max(...timestamps)).toISOString(),
executionTimeRange: {
min: Math.min(...executionDurations),
max: Math.max(...executionDurations),
avg: executionDurations.reduce((a, b) => a + b, 0) / executionDurations.length
}
};
return chainSummary;
}
catch (error) {
console.error('Error getting chain summary:', error);
return null;
}
}
}
exports.ExecutionTracker = ExecutionTracker;
//# sourceMappingURL=execution-tracker.js.map