pit-manager
Version:
Centralized prompt management system for Human Behavior AI agents
603 lines • 23.1 kB
JavaScript
"use strict";
/**
* Unified model interface with automatic tracking and chaining for TypeScript
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.model = exports.Model = exports.ModelResponse = void 0;
const path = __importStar(require("path"));
const crypto = __importStar(require("crypto"));
const registry_1 = require("../providers/registry");
const structured_output_1 = require("./structured-output");
const execution_tracker_1 = require("../storage/execution-tracker");
const prompts_1 = require("./prompts");
/**
* ModelResponse implementation that enables automatic chaining
*/
class ModelResponse {
content;
model;
promptHash;
executionId;
metadata;
rawResponse;
_isPitResponse = true;
constructor(params) {
this.content = params.content;
this.model = params.model;
this.promptHash = params.promptHash;
this.executionId = params.executionId;
this.metadata = params.metadata;
this.rawResponse = params.rawResponse;
}
/**
* String representation returns the content
*/
toString() {
if (typeof this.content === 'object') {
return JSON.stringify(this.content, null, 2);
}
return String(this.content);
}
/**
* Convert response to prompt string for chaining
*/
toPrompt() {
return this.toString();
}
/**
* Concatenate with another ModelResponse or string
*/
plus(other) {
return this.concatenate(other);
}
/**
* Internal concatenation logic
*/
concatenate(other) {
// Convert string to ModelResponse if needed
if (typeof other === 'string') {
other = ModelResponse.fromString(other);
}
// Check for duplicate concatenation
if (this.executionId === other.executionId) {
console.warn(`⚠️ Duplicate concatenation detected: ${this.executionId}`);
return this;
}
// Combine agent names intelligently
const selfTag = this.metadata.tag;
const otherTag = other.metadata.tag;
const combinedTag = this.combineAgentNames(selfTag, otherTag);
// Generate deterministic chain ID based on unique agents
const agentSequence = this.getAgentSequence().concat(other.getAgentSequence());
const newChainId = this.generateDeterministicChainId(agentSequence);
// Combine content
const combinedContent = this.toString() + '\n' + other.toString();
// Create new execution ID
const newExecutionId = crypto
.createHash('sha256')
.update(`${combinedContent}_${Date.now()}`)
.digest('hex');
return new ModelResponse({
content: combinedContent,
model: `${this.model}+${other.model}`,
promptHash: crypto.createHash('sha256').update(combinedContent).digest('hex'),
executionId: newExecutionId,
metadata: {
tag: combinedTag,
provider: `${this.metadata.provider}+${other.metadata.provider}`,
chainId: newChainId,
chainGroupId: this.metadata.chainGroupId || other.metadata.chainGroupId, // Preserve chain group ID
tokens: {
prompt: this.metadata.tokens.prompt + other.metadata.tokens.prompt,
completion: this.metadata.tokens.completion + other.metadata.tokens.completion,
total: this.metadata.tokens.total + other.metadata.tokens.total
},
latencyMs: Math.max(this.metadata.latencyMs, other.metadata.latencyMs),
structured: this.metadata.structured || other.metadata.structured,
concatenated: true,
originalResponses: [this.executionId, other.executionId],
agentSequence,
timestamp: new Date().toISOString()
}
});
}
/**
* Combine agent names intelligently, avoiding duplicates
*/
combineAgentNames(tag1, tag2) {
// If both tags are the same, return just one
if (tag1 === tag2) {
return tag1;
}
// Parse existing combined tags
const agents1 = tag1.includes('+') ? tag1.split('+') : [tag1];
const agents2 = tag2.includes('+') ? tag2.split('+') : [tag2];
// Get unique agents while preserving order
const seen = new Set();
const uniqueAgents = [];
for (const agent of [...agents1, ...agents2]) {
if (!seen.has(agent)) {
seen.add(agent);
uniqueAgents.push(agent);
}
}
return uniqueAgents.join('+');
}
/**
* Get the sequence of agents in this response
*/
getAgentSequence() {
const metadata = this.metadata;
if (metadata.agentSequence) {
return metadata.agentSequence;
}
const tag = this.metadata.tag;
return tag.includes('+') ? tag.split('+') : [tag];
}
/**
* Generate deterministic chain ID from agent sequence
*/
generateDeterministicChainId(agentSequence) {
// Get unique agents and sort for consistency
const uniqueAgents = [...new Set(agentSequence)].sort();
const signature = uniqueAgents.join('->');
// Generate deterministic hash
const hash = crypto.createHash('sha256').update(signature).digest('hex');
return `auto_${hash.substring(0, 12)}`;
}
/**
* Create ModelResponse from string
*/
static fromString(content, tag = 'string-context') {
const executionId = crypto
.createHash('sha256')
.update(`${content}_${Date.now()}`)
.digest('hex');
return new ModelResponse({
content,
model: 'string',
promptHash: crypto.createHash('sha256').update(content).digest('hex'),
executionId,
metadata: {
tag,
provider: 'string',
tokens: { prompt: 0, completion: 0, total: 0 },
latencyMs: 0,
structured: false,
fromString: true,
timestamp: new Date().toISOString()
}
});
}
}
exports.ModelResponse = ModelResponse;
/**
* Unified model interface with automatic tracking
*/
class Model {
repoPath = null;
versioning = null;
executionTracker = null;
registry;
constructor() {
this.registry = new registry_1.ProviderRegistry();
this.initializeFromEnvironment();
}
/**
* Start a new chain execution group for process-unique tracking.
*/
startChainExecutionGroup(chainId, options) {
if (!this.executionTracker) {
throw new Error('Execution tracker not available');
}
const sessionId = options?.session_id;
const processId = options?.process_id || process.pid.toString();
const metadata = options?.metadata;
return this.executionTracker.startChainExecutionGroup(chainId, sessionId, processId, metadata);
}
/**
* End a chain execution group.
*/
endChainExecutionGroup(groupId, status = 'completed') {
if (this.executionTracker) {
this.executionTracker.endChainExecutionGroup(groupId, status);
}
}
/**
* Get the active chain group ID for the current process.
*/
getActiveChainGroupId() {
if (this.executionTracker) {
return this.executionTracker.getActiveChainGroupId();
}
return undefined;
}
/**
* Execute model with automatic tracking and optional structured output
*/
async complete(modelName, prompt, tag, jsonOutput, options) {
const startTime = Date.now();
// Capture prompt file context before clearing
const promptFile = (0, prompts_1.getCurrentPromptFile)();
// Detect provider from model name
const providerName = this.detectProvider(modelName);
// Get provider
const provider = this.registry.getProvider(providerName);
if (!provider) {
throw new Error(`Provider not found: ${providerName}`);
}
// Prepare prompt content
let promptStr;
let parentExecutionId;
let chainId;
let chainGroupId;
if (prompt instanceof ModelResponse) {
// This is a chained call
promptStr = prompt.toPrompt();
parentExecutionId = prompt.executionId;
chainId = prompt.metadata.chainId;
chainGroupId = prompt.metadata.chainGroupId;
}
else if (typeof prompt === 'string') {
promptStr = prompt;
// Check if prompts() detected a parent execution ID
parentExecutionId = (0, prompts_1.getParentExecutionId)();
}
else if (typeof prompt === 'object' && 'content' in prompt) {
promptStr = typeof prompt.content === 'string' ? prompt.content : JSON.stringify(prompt.content);
parentExecutionId = (0, prompts_1.getParentExecutionId)();
}
else if (typeof prompt === 'object' && 'parts' in prompt) {
// Handle MultimodalContent
const multimodalContent = prompt;
promptStr = multimodalContent.parts
.filter((part) => part.type === 'text')
.map((part) => part.text)
.join('\n');
parentExecutionId = (0, prompts_1.getParentExecutionId)();
}
else {
throw new Error('Invalid prompt type');
}
// Override with explicit options if provided
if (options?.parentExecutionId) {
console.log(`[PIT-DEBUG] Setting parentExecutionId from options: ${options.parentExecutionId}`);
parentExecutionId = options.parentExecutionId;
}
if (options?.chainId) {
chainId = options.chainId;
}
if (options?.chainGroupId) {
chainGroupId = options.chainGroupId;
}
// Use active chain group if none provided
if (!chainGroupId && this.executionTracker) {
chainGroupId = this.executionTracker.getActiveChainGroupId();
}
// Prepare structured output if needed
let structuredConfig = {};
if (jsonOutput) {
const handler = new structured_output_1.StructuredOutputHandler();
if (providerName === 'openai') {
structuredConfig = handler.prepareOpenAIStructured(jsonOutput);
}
else if (providerName === 'google') {
structuredConfig = handler.prepareGeminiStructured(jsonOutput);
}
else if (providerName === 'anthropic') {
structuredConfig = handler.prepareClaudeStructured(jsonOutput);
}
}
// Create messages
const messages = [{ role: 'user', content: promptStr }];
// Execute with provider
let response;
const cleanOptions = { ...options };
delete cleanOptions.parentExecutionId;
delete cleanOptions.chainId;
delete cleanOptions.chainGroupId;
if (providerName === 'openai' && jsonOutput && structuredConfig.useBeta) {
// Use OpenAI standard API for structured output (beta API structure changed)
const { OpenAI } = require('openai');
const client = new OpenAI({ apiKey: provider.apiKey });
// Clean structured config - remove internal PIT parameters
const openaiConfig = { ...structuredConfig };
delete openaiConfig.useBeta; // Remove useBeta flag
response = await client.chat.completions.create({
model: modelName,
messages: [{ role: 'user', content: promptStr }],
response_format: openaiConfig.response_format,
...cleanOptions,
...openaiConfig
});
}
else {
// Regular API call
response = await provider.chatCompletion(modelName, messages, {
...cleanOptions,
...structuredConfig
});
}
// Parse structured output if needed
let content;
if (jsonOutput) {
const handler = new structured_output_1.StructuredOutputHandler();
if (providerName === 'openai') {
content = handler.parseOpenAIResponse(response, jsonOutput);
}
else if (providerName === 'google') {
content = handler.parseGeminiResponse(response, jsonOutput);
}
else if (providerName === 'anthropic') {
content = handler.parseClaudeResponse(response, jsonOutput);
}
else {
content = this.extractContent(response);
}
}
else {
// Regular text response
content = this.extractContent(response);
}
// Calculate metrics
const endTime = Date.now();
const latencyMs = endTime - startTime;
// Extract token usage
const tokens = this.extractTokens(response, providerName);
if (process.env.PIT_DEBUG) {
console.debug('Token extraction:', { provider: providerName, response: response.usage, tokens });
}
// Track execution
const promptHash = crypto.createHash('sha256').update(promptStr).digest('hex');
const executionId = crypto.createHash('sha256')
.update(`${promptHash}_${new Date().toISOString()}`)
.digest('hex');
// Create or update chain
if (!chainId && parentExecutionId) {
// Try to inherit chainId from parent execution
if (this.executionTracker) {
try {
const parentExecution = await this.executionTracker.getExecution(parentExecutionId);
if (parentExecution && parentExecution.chain_id) {
chainId = parentExecution.chain_id;
}
}
catch (error) {
// If we can't find parent, create new chain
if (process.env.PIT_DEBUG) {
console.debug('Could not inherit chainId from parent:', error);
}
}
}
// If still no chainId, create a new one
if (!chainId) {
chainId = crypto.createHash('sha256')
.update(`${tag}_${new Date().toISOString()}`)
.digest('hex');
}
}
// Track execution if tracker available
if (this.executionTracker) {
// Use the captured prompt file (already captured before clearing context)
const cost = this.calculateCost(tokens, modelName, providerName);
await this.executionTracker.trackExecution(executionId, modelName, promptHash, tag, tokens, latencyMs, cost, chainId || undefined, parentExecutionId || undefined, {
provider: providerName,
structured_output: jsonOutput !== undefined,
prompt_file: promptFile
}, chainGroupId // NEW: Pass chain group ID
);
// Auto-commit if not in active chain
if (!parentExecutionId && this.versioning) {
try {
await this.versioning.commit(`Model execution: ${tag} with ${modelName}`, 'PIT Auto-Tracker');
}
catch (error) {
if (process.env.PIT_DEBUG) {
console.debug('Auto-commit failed:', error);
}
}
}
}
// Clear prompt context after tracking
(0, prompts_1.clearPromptContext)();
// Create response object
return new ModelResponse({
content,
model: modelName,
promptHash,
executionId,
metadata: {
tag,
provider: providerName,
chainId,
chainGroupId, // NEW: Include in metadata
tokens,
latencyMs,
structured: jsonOutput !== undefined
},
rawResponse: response
});
}
initializeFromEnvironment() {
// Load environment variables
try {
require('dotenv').config();
}
catch {
// dotenv not available, continue without it
}
// Find .pit directory
this.repoPath = this.findPitDirectory();
if (this.repoPath) {
try {
// Initialize versioning
const { VersioningEngine } = require('../versioning/operations');
this.versioning = new VersioningEngine(this.repoPath);
this.executionTracker = new execution_tracker_1.ExecutionTracker(this.repoPath);
if (process.env.PIT_DEBUG) {
console.debug('PIT initialized with repo path:', this.repoPath);
}
}
catch (error) {
if (process.env.PIT_DEBUG) {
console.debug('Failed to initialize PIT components:', error);
}
}
}
}
findPitDirectory() {
let current = process.cwd();
while (current !== path.dirname(current)) {
const pitDir = path.join(current, '.pit');
try {
if (require('fs').statSync(pitDir).isDirectory()) {
return pitDir;
}
}
catch {
// Directory doesn't exist
}
current = path.dirname(current);
}
// Check current directory one more time
const pitDir = path.join(process.cwd(), '.pit');
try {
if (require('fs').statSync(pitDir).isDirectory()) {
return pitDir;
}
}
catch {
// Directory doesn't exist
}
return null;
}
detectProvider(modelName) {
const modelLower = modelName.toLowerCase();
if (modelLower.includes('gpt') || modelLower.includes('o1') || modelLower.includes('davinci') ||
modelLower.includes('curie') || modelLower.includes('babbage') || modelLower.includes('ada')) {
return 'openai';
}
else if (modelLower.includes('gemini') || modelLower.includes('palm') || modelLower.includes('bison')) {
return 'google';
}
else if (modelLower.includes('claude') || modelLower.includes('anthropic')) {
return 'anthropic';
}
else {
// Default to OpenAI for unknown models
return 'openai';
}
}
extractContent(response) {
if (response.content) {
return response.content;
}
else if (response.choices && response.choices[0] && response.choices[0].message) {
return response.choices[0].message.content;
}
else if (response.text) {
return response.text;
}
else {
return String(response);
}
}
extractTokens(response, _provider) {
const tokens = { prompt: 0, completion: 0, total: 0 };
// Check if response has tokens field (from providers)
if (response.tokens) {
const usage = response.tokens;
if (usage.input !== undefined) {
tokens.prompt = usage.input;
}
if (usage.output !== undefined) {
tokens.completion = usage.output;
}
if (usage.total !== undefined) {
tokens.total = usage.total;
}
else {
tokens.total = tokens.prompt + tokens.completion;
}
}
// Fallback to usage field for direct OpenAI responses
else if (response.usage) {
const usage = response.usage;
if (usage.prompt_tokens !== undefined) {
tokens.prompt = usage.prompt_tokens;
}
if (usage.completion_tokens !== undefined) {
tokens.completion = usage.completion_tokens;
}
if (usage.total_tokens !== undefined) {
tokens.total = usage.total_tokens;
}
else {
tokens.total = tokens.prompt + tokens.completion;
}
}
return tokens;
}
calculateCost(tokens, model, _provider) {
// Simplified cost calculation
const totalTokens = tokens.total;
// Rough cost estimates (USD per 1K tokens)
const costPer1k = {
'gpt-4': 0.03,
'gpt-4o': 0.005,
'gpt-3.5-turbo': 0.002,
'claude-3-5-sonnet': 0.003,
'claude-3-haiku': 0.00025,
'gemini-2.0-flash': 0.00075,
'gemini-1.5-pro': 0.0035,
};
const baseCost = costPer1k[model] || 0.01;
return (totalTokens / 1000) * baseCost;
}
/**
* Get aggregated chain execution summary
*/
async getChainSummary(chainId) {
if (!this.executionTracker) {
throw new Error('Execution tracking not available - no .pit directory found');
}
return await this.executionTracker.getChainSummary(chainId);
}
}
exports.Model = Model;
// Export singleton instance
exports.model = new Model();
//# sourceMappingURL=model.js.map