json-object-editor
Version:
JOE the Json Object Editor | Platform Edition
586 lines (542 loc) • 21.8 kB
JavaScript
const OpenAI = require('openai');
/**
* ThoughtPipeline
*
* Minimal pipeline + agent runner for the `thought` itemtype.
* - Pipelines compile deterministic, structured context from JOE data.
* - Agents call the OpenAI Responses API with that context and materialize
* proposed Thought objects plus an ai_response record.
*
* This is intentionally small and code-configured for MVP; later it can be
* backed by JOE schemas if needed.
*/
const ThoughtPipeline = {};
// -----------------------
// CONFIG: PIPELINES
// -----------------------
// For MVP we define a single built-in pipeline. Later this can be driven
// by a `pipeline` schema if you want full UI editing.
const PIPELINES = {
'thought_default': {
id: 'thought_default',
name: 'Default Thought Pipeline',
steps: [
{
id: 'schema_summaries',
step_type: 'schema_summaries',
selector: {
// Keep this small and explicit for determinism
names: ['thought', 'ai_prompt', 'ai_response']
},
render_mode: 'json',
required: true
},
{
id: 'accepted_thoughts',
step_type: 'thoughts',
selector: {
// Deterministic query: only accepted thoughts, newest first
query: { itemtype: 'thought', status: 'accepted' },
sortBy: 'joeUpdated',
sortDir: 'desc'
},
render_mode: 'bullets',
required: false
}
// Additional steps (e.g., tools/objects) can be added later.
]
}
};
// -----------------------
// CONFIG: AGENTS
// -----------------------
// Simple, code-based agent configs keyed by id.
const AGENTS = {
'thought_agent_default': {
id: 'thought_agent_default',
name: 'Default Thought Agent',
pipeline_id: 'thought_default',
model: 'gpt-4.1-mini',
system_prompt: [
'You are JOE (Json Object Editor) Thought Engine.',
'You receive:',
'- A compiled context that ALWAYS includes a `scope_object` section describing the primary object for this run.',
'- Additional JOE context (schemas, accepted thoughts, and related metadata).',
'- A natural language instruction or question from a human.',
'',
'Your job:',
'- Propose new Thought objects that help model the world: hypotheses, syntheses, and links.',
'- Each proposed thought MUST follow the `thought` schema:',
' - thought_type: hypothesis | synthesis | link',
' - statement: short, human-readable',
' - certainty: numeric 0–1 (float)',
' - about[]: objects/schemas it is about (each: { itemtype, id, role?, note? })',
' - because[]: supporting receipts (same shape as about[])',
' - For link thoughts only: a{itemtype,id}, b{itemtype,id}, relationship_type, rationale.',
'',
'Scope rules:',
'- Unless the user explicitly requests meta-thoughts, every proposed Thought must be about the domain of the scope object, not about prompts, schemas, or the act of generating thoughts.',
'- When a concrete scope_object is provided, at least one `about[]` entry SHOULD reference it directly:',
' - { itemtype: scope_object.itemtype, id: scope_object._id, role: "subject" }',
'',
'Quality rules:',
'- Only propose thoughts that are grounded in the provided context.',
'- Prefer a small number of high-quality thoughts (at most 3) over many weak ones.',
'- If you lack evidence, lower certainty or omit the thought.',
'',
'Output format:',
'- Return strict JSON with top-level keys:',
' {',
' "proposed_thoughts": [Thought, ...],',
' "questions": [string, ...],',
' "missing_info": [string, ...]',
' }',
'- Do NOT include explanations outside the JSON.'
].join('\n'),
policies: {
must_propose_structured_thoughts: true,
must_use_because_for_hypotheses_and_links: true
},
output_contract: {
keys: ['proposed_thoughts', 'questions', 'missing_info']
}
}
};
// -----------------------
// INTERNAL HELPERS
// -----------------------
function getAPIKey() {
if (!global.JOE || !JOE.Utils || typeof JOE.Utils.Settings !== 'function') {
throw new Error('JOE Utils/Settings not initialized');
}
const key = JOE.Utils.Settings('OPENAI_API_KEY');
if (!key) {
throw new Error('Missing OPENAI_API_KEY setting');
}
return key;
}
function getPipeline(pipelineId) {
var id = pipelineId || 'thought_default';
var pipe = PIPELINES[id];
if (!pipe) {
throw new Error('Unknown pipeline_id: ' + id);
}
return pipe;
}
function getAgent(agentId) {
var id = agentId || 'thought_agent_default';
var agent = AGENTS[id];
if (!agent) {
throw new Error('Unknown agent_id: ' + id);
}
return agent;
}
function getSchemaSummary(name) {
if (!name || !global.JOE || !JOE.Schemas) return null;
var Schemas = JOE.Schemas;
if (Schemas.summary && Schemas.summary[name]) {
return Schemas.summary[name];
}
if (Schemas.schema && Schemas.schema[name] && Schemas.schema[name].summary) {
return Schemas.schema[name].summary;
}
return null;
}
function sortByField(items, field, dir) {
if (!Array.isArray(items)) return [];
var direction = (dir === 'asc') ? 1 : -1;
return items.slice().sort(function (a, b) {
var av = a && a[field];
var bv = b && b[field];
if (av == null && bv == null) return 0;
if (av == null) return 1;
if (bv == null) return -1;
if (av > bv) return 1 * direction;
if (av < bv) return -1 * direction;
return 0;
});
}
// Best-effort helper to get objects from cache in a deterministic way.
function queryObjects(selector) {
if (!global.JOE || !JOE.Cache || typeof JOE.Cache.search !== 'function') {
throw new Error('JOE Cache not initialized');
}
selector = selector || {};
var q = selector.query || {};
var items = JOE.Cache.search(q) || [];
// Optionally filter by itemtype to be safe.
if (selector.itemtype) {
items = items.filter(function (it) {
return it && it.itemtype === selector.itemtype;
});
}
// Stable sort
if (selector.sortBy) {
items = sortByField(items, selector.sortBy, selector.sortDir || 'desc');
} else if (q.itemtype) {
items = sortByField(items, 'joeUpdated', 'desc');
}
return items;
}
// Render a list of thought objects as simple bullets for prompts.
function renderThoughtBullets(items) {
if (!Array.isArray(items) || !items.length) return '';
return items.map(function (t) {
var type = t.thought_type || '';
var status = t.status || '';
var certainty = (typeof t.certainty === 'number')
? (Math.round(t.certainty * 100) + '%')
: (t.certainty || '');
var bits = [];
type && bits.push(type);
status && bits.push(status);
certainty && bits.push(certainty);
var meta = bits.length ? ('[' + bits.join(' | ') + '] ') : '';
return '- ' + meta + (t.statement || '');
}).join('\n');
}
// Render a pipeline step into text for use in a prompt, plus keep raw data.
function renderStep(step, data) {
var mode = step.render_mode || 'json';
if (mode === 'bullets' && step.step_type === 'thoughts') {
return renderThoughtBullets(data || []);
}
if (mode === 'json') {
try {
return JSON.stringify(data, null, 2);
} catch (e) {
return '';
}
}
// compact_json: no pretty-print
if (mode === 'compact_json') {
try {
return JSON.stringify(data);
} catch (e) {
return '';
}
}
// text/template modes could be added later; for now treat as plain string.
if (mode === 'text' && typeof data === 'string') {
return data;
}
return '';
}
// Extract JSON text from a model output string (copied in spirit from chatgpt.js).
function extractJsonText(raw) {
if (!raw) { return ''; }
var t = String(raw).trim();
var fenceIdx = t.indexOf('```json') !== -1 ? t.indexOf('```json') : t.indexOf('```');
if (fenceIdx !== -1) {
var start = fenceIdx;
var firstNewline = t.indexOf('\n', start);
if (firstNewline !== -1) {
t = t.substring(firstNewline + 1);
} else {
t = t.substring(start + 3);
}
var lastFence = t.lastIndexOf('```');
if (lastFence !== -1) {
t = t.substring(0, lastFence);
}
t = t.trim();
}
if (t[0] !== '{' && t[0] !== '[') {
var firstBrace = t.indexOf('{');
var firstBracket = t.indexOf('[');
var first = -1;
if (firstBrace === -1) { first = firstBracket; }
else if (firstBracket === -1) { first = firstBrace; }
else { first = Math.min(firstBrace, firstBracket); }
var lastBrace = Math.max(t.lastIndexOf('}'), t.lastIndexOf(']'));
if (first !== -1 && lastBrace !== -1 && lastBrace > first) {
t = t.slice(first, lastBrace + 1);
}
}
return t.trim();
}
// -----------------------
// PUBLIC: PIPELINE
// -----------------------
/**
* Compile a pipeline into a deterministic prompt payload.
*
* @param {string} pipelineId
* @param {string} scopeId - optional object id the pipeline is scoped around
* @param {object} opts - { user_input?: string }
*/
ThoughtPipeline.compile = async function compile(pipelineId, scopeId, opts) {
var pipeline = getPipeline(pipelineId);
opts = opts || {};
var compiledSteps = [];
var scopeStep = null;
// If we have a scope object, include a dedicated scope_object step first.
if (scopeId && global.JOE && JOE.Cache && typeof JOE.Cache.findByID === 'function') {
try {
var base = JOE.Cache.findByID(scopeId) || null;
if (base) {
var flat = null;
if (JOE.Utils && typeof JOE.Utils.flattenObject === 'function') {
try {
flat = JOE.Utils.flattenObject(base._id, { recursive: true, depth: 2 });
} catch (_e) { flat = null; }
}
var sSummary = getSchemaSummary(base.itemtype);
var payload = {
scope_object: {
_id: base._id,
itemtype: base.itemtype,
object: base,
flattened: flat || null,
schema_summary: sSummary || null
}
};
scopeStep = {
id: 'scope_object',
step_type: 'scope_object',
render_mode: 'json',
text: JSON.stringify(payload, null, 2),
data: payload
};
compiledSteps.push(scopeStep);
}
} catch (_e) { /* best-effort only */ }
}
for (var i = 0; i < pipeline.steps.length; i++) {
var step = pipeline.steps[i];
var data = null;
if (step.step_type === 'schema_summaries') {
var names = (step.selector && step.selector.names) || [];
data = {};
names.forEach(function (n) {
var sum = getSchemaSummary(n);
if (sum) {
data[n] = sum;
}
});
} else if (step.step_type === 'thoughts') {
var selector = step.selector || {};
// Ensure itemtype thought by default
if (!selector.itemtype) {
selector.itemtype = 'thought';
}
data = queryObjects(selector);
} else if (step.step_type === 'objects' || step.step_type === 'tools') {
data = queryObjects(step.selector || {});
} else if (step.step_type === 'text') {
data = (step.selector && step.selector.text) || '';
}
if (!data && step.required) {
throw new Error('Required pipeline step failed: ' + (step.id || step.step_type));
}
var text = renderStep(step, data);
compiledSteps.push({
id: step.id || step.step_type,
step_type: step.step_type,
render_mode: step.render_mode || 'json',
text: text,
data: data
});
}
// For MVP, put most context into system_context as structured sections.
var systemParts = compiledSteps.map(function (s) {
return '### Step: ' + s.id + '\n' + s.text;
});
var compiled = {
pipeline_id: pipeline.id,
scope_id: scopeId || null,
system_context: systemParts.join('\n\n'),
developer_context: '',
user_context: opts.user_input || '',
attachments: {
steps: compiledSteps
}
};
return compiled;
};
// -----------------------
// PUBLIC: AGENT RUNNER
// -----------------------
/**
* Run an agent against a pipeline and materialize proposed Thoughts.
*
* @param {string} agentId
* @param {string} userInput
* @param {string} scopeId
* @param {object} ctx - optional context (e.g., { req })
*/
ThoughtPipeline.runAgent = async function runAgent(agentId, userInput, scopeId, ctx) {
var agent = getAgent(agentId);
var compiled = await ThoughtPipeline.compile(agent.pipeline_id, scopeId, { user_input: userInput || '' });
var apiKey = getAPIKey();
var openai = new OpenAI({ apiKey: apiKey });
var response = await openai.responses.create({
model: agent.model || 'gpt-4.1-mini',
instructions: agent.system_prompt,
input: JSON.stringify({
pipeline_id: compiled.pipeline_id,
scope_id: compiled.scope_id,
compiled_context: compiled,
user_input: userInput || '',
policies: agent.policies || {}
}, null, 2),
temperature: 0.2
});
var rawText = '';
try {
rawText = response.output_text || '';
} catch (e) {
rawText = '';
}
var parsed = { proposed_thoughts: [], questions: [], missing_info: [] };
try {
var jsonText = extractJsonText(rawText);
if (jsonText) {
var obj = JSON.parse(jsonText);
parsed.proposed_thoughts = Array.isArray(obj.proposed_thoughts) ? obj.proposed_thoughts : [];
parsed.questions = Array.isArray(obj.questions) ? obj.questions : [];
parsed.missing_info = Array.isArray(obj.missing_info) ? obj.missing_info : [];
}
} catch (e) {
// swallow parse errors; we still persist the ai_response below
}
// Persist ai_response for traceability
var aiResponseObj = {
itemtype: 'ai_response',
name: agent.name + ' → Thoughts',
response_type: 'thought_generation',
response: rawText || '',
response_id: response.id || '',
user_prompt: userInput || '',
model_used: agent.model || 'gpt-4.1-mini',
tags: [],
referenced_objects: scopeId ? [scopeId] : [],
usage: response.usage || {},
prompt_method: 'ThoughtPipeline.runAgent'
};
var savedResponse = await new Promise(function (resolve, reject) {
try {
JOE.Storage.save(aiResponseObj, 'ai_response', function (err, saved) {
if (err) return reject(err);
resolve(saved);
}, {
user: (ctx && ctx.req && ctx.req.User) || { name: 'thought_agent' },
history: true
});
} catch (e) {
reject(e);
}
});
var savedResponseId = savedResponse && savedResponse._id;
// Resolve default status for Thoughts (dataset includes 'thought')
var defaultThoughtStatusId = null;
try {
if (JOE && JOE.Data && Array.isArray(JOE.Data.status)) {
var allStatus = JOE.Data.status;
var match = allStatus.find(function (s) {
return Array.isArray(s.datasets) && s.datasets.includes('thought') && s.default;
}) || allStatus.find(function (s) {
return Array.isArray(s.datasets) && s.datasets.includes('thought') && s.active;
}) || null;
if (match && match._id) {
defaultThoughtStatusId = match._id;
}
}
} catch (_e) { /* best-effort only */ }
// Materialize proposed_thoughts as `thought` objects
var nowIso = (new Date()).toISOString();
var thoughtObjects = (parsed.proposed_thoughts || []).map(function (t) {
var certainty = t.certainty;
if (typeof certainty === 'string') {
var num = parseFloat(certainty);
if (!isNaN(num)) certainty = num;
}
var baseName = (t.name || t.statement || 'Thought').trim();
if (baseName.length > 32) {
baseName = baseName.slice(0, 29);
var lastSpace = baseName.lastIndexOf(' ');
if (lastSpace > 10) {
baseName = baseName.slice(0, lastSpace);
}
baseName += '…';
}
var name = baseName;
return {
itemtype: 'thought',
name: name,
thought_type: t.thought_type || 'hypothesis',
statement: t.statement || '',
certainty: (typeof certainty === 'number') ? certainty : null,
status: defaultThoughtStatusId || null,
tags: Array.isArray(t.tags) ? t.tags : [],
about: Array.isArray(t.about) ? t.about : [],
because: Array.isArray(t.because) ? t.because : [],
a: t.a || null,
b: t.b || null,
relationship_type: t.relationship_type || null,
rationale: t.rationale || null,
lineage: {
ai_response_id: savedResponseId,
created_by: 'agent:' + agent.id
},
source_ai_response: savedResponseId,
created_by: 'agent:' + agent.id
};
}).filter(function (obj) {
return obj.statement && obj.statement.length;
});
var savedThoughts = [];
if (thoughtObjects.length) {
savedThoughts = await new Promise(function (resolve, reject) {
try {
JOE.Storage.save(thoughtObjects, 'thought', function (err, saved) {
if (err) return reject(err);
resolve(saved || []);
}, {
user: (ctx && ctx.req && ctx.req.User) || { name: 'thought_agent' },
history: true
});
} catch (e) {
reject(e);
}
});
// Normalize to an array; Storage.save may return a single object or an array
var savedThoughtArray = Array.isArray(savedThoughts)
? savedThoughts
: (savedThoughts ? [savedThoughts] : []);
// Back-link generated thoughts onto the ai_response
if (savedResponseId && savedThoughtArray.length) {
var thoughtIds = savedThoughtArray.map(function (t) { return t && t._id; }).filter(Boolean);
if (thoughtIds.length) {
try {
var updateResp = Object.assign({}, savedResponse, {
generated_thoughts: thoughtIds,
joeUpdated: new Date().toISOString()
});
await new Promise(function (resolve, reject) {
JOE.Storage.save(updateResp, 'ai_response', function (err, saved) {
if (err) return reject(err);
resolve(saved);
}, {
user: (ctx && ctx.req && ctx.req.User) || { name: 'thought_agent' },
history: true
});
});
} catch (_e) { /* best-effort only */ }
}
}
}
return {
agent_id: agent.id,
pipeline_id: agent.pipeline_id,
scope_id: scopeId || null,
ai_response_id: savedResponseId || null,
proposed_thoughts_count: thoughtObjects.length,
saved_thought_ids: (Array.isArray(savedThoughts) ? savedThoughts : (savedThoughts ? [savedThoughts] : []))
.map(function (t) { return t && t._id; }).filter(Boolean),
questions: parsed.questions || [],
missing_info: parsed.missing_info || [],
raw_response: rawText
};
};
module.exports = ThoughtPipeline;