mcard-js
Version:
MCard - Content-addressable storage with cryptographic hashing, handle resolution, and vector search for Node.js and browsers
1,464 lines (1,453 loc) • 52.3 kB
JavaScript
import {
MCard
} from "./chunk-PW4XS7M3.js";
import "./chunk-7NKII2JA.js";
import "./chunk-MLKGABMK.js";
// src/ptr/node/NetworkRuntime.ts
import * as http from "http";
// src/ptr/node/P2PChatSession.ts
var P2PChatSession = class {
sessionId;
collection;
buffer = [];
previousHash = null;
sequence = 0;
maxBufferSize;
constructor(collection, sessionId, maxBufferSize = 5, initialHeadHash = null) {
this.collection = collection;
this.sessionId = sessionId;
this.maxBufferSize = maxBufferSize;
this.previousHash = initialHeadHash;
if (initialHeadHash) {
}
}
/**
* Add a message to the current session buffer.
* Automatically checkpoints if buffer exceeds size.
*/
async addMessage(sender, content) {
this.buffer.push({
sender,
content,
timestamp: Date.now()
});
if (this.buffer.length >= this.maxBufferSize) {
return this.checkpoint();
}
return null;
}
/**
* Force write the current buffer to a new MCard.
*/
async checkpoint() {
if (this.buffer.length === 0) {
return this.previousHash || "";
}
const payload = {
type: "p2p_session_segment",
sessionId: this.sessionId,
sequence: this.sequence++,
messages: [...this.buffer],
previousHash: this.previousHash,
timestamp: Date.now()
};
const card = await MCard.create(JSON.stringify(payload));
await this.collection.add(card);
this.previousHash = card.hash;
this.buffer = [];
console.log(`[P2PSession] Checkpoint created: ${card.hash} (Seq: ${payload.sequence})`);
return card.hash;
}
/**
* Get the hash of the latest segment (Head of the list)
*/
getHeadHash() {
return this.previousHash;
}
/**
* Compile all segments into one MCard and remove original segments unless keepOriginals is true.
*/
async summarize(keepOriginals = false) {
if (this.buffer.length > 0) {
await this.checkpoint();
}
const headToUse = this.previousHash || null;
console.log(`[P2PSession] Summarizing session starting from head: ${headToUse}`);
const { messages, hashes } = headToUse ? await this.traverseChain(headToUse) : { messages: [], hashes: [] };
const summaryPayload = {
type: "p2p_session_summary",
sessionId: this.sessionId,
originalHeadHash: headToUse,
// Cast or update interface
fullTranscript: messages,
timestamp: Date.now()
};
const summaryContent = JSON.stringify(summaryPayload, null, 2);
const summaryCard = await MCard.create(summaryContent);
await this.collection.add(summaryCard);
console.log(`[P2PSession] Summary created: ${summaryCard.hash}`);
if (!keepOriginals) {
console.log(`[P2PSession] Cleaning up ${hashes.length} segment MCards...`);
for (const hash of hashes) {
try {
await this.collection.delete(hash);
} catch (e) {
console.error(`[P2PSession] Failed to delete segment ${hash}`, e);
}
}
console.log(`[P2PSession] Cleanup complete.`);
} else {
console.log(`[P2PSession] Skipping cleanup (keepOriginals=true). Preserved ${hashes.length} segments.`);
}
return summaryCard.hash;
}
async traverseChain(headHash) {
const messages = [];
const hashes = [];
let currentHash = headHash;
while (currentHash) {
hashes.push(currentHash);
const card = await this.collection.get(currentHash);
if (!card) {
console.warn(`[P2PSession] Broken chain at ${currentHash}`);
break;
}
try {
const contentStr = new TextDecoder().decode(card.content);
const payload = JSON.parse(contentStr);
if (payload.type === "p2p_session_segment") {
messages.unshift(...payload.messages);
currentHash = payload.previousHash;
} else {
console.warn(`[P2PSession] Invalid card type at ${currentHash}`);
break;
}
} catch (e) {
console.error(`[P2PSession] Parse error at ${currentHash}`, e);
break;
}
}
return { messages, hashes };
}
};
// src/ptr/node/SignalingServer.ts
import { createServer } from "http";
import { URL as URL2 } from "url";
import { exec } from "child_process";
function killProcessOnPort(port) {
return new Promise((resolve) => {
exec(`lsof -ti:${port} | xargs kill -9 2>/dev/null`, (error) => {
if (error) {
console.log(`[Signal] No existing process on port ${port} to kill`);
} else {
console.log(`[Signal] Killed existing process on port ${port}`);
}
setTimeout(resolve, 500);
});
});
}
async function createSignalingServer(config = {}) {
const startPort = config.port || 3e3;
const maxTries = config.maxPortTries || 10;
const autoFindPort = config.autoFindPort !== false;
const clients = /* @__PURE__ */ new Map();
const messageBuffer = /* @__PURE__ */ new Map();
const server = createServer((req, res) => {
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
if (req.method === "OPTIONS") {
res.writeHead(204);
res.end();
return;
}
const url = new URL2(req.url || "/", `http://${req.headers.host}`);
const path = url.pathname;
if (req.method === "GET" && path === "/health") {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({
status: "ok",
clients: Array.from(clients.keys()),
buffered: Array.from(messageBuffer.keys())
}));
return;
}
if (req.method === "GET" && path === "/signal") {
const peerId = url.searchParams.get("peer_id");
if (!peerId) {
res.writeHead(400);
res.end("Missing peer_id");
return;
}
console.log(`[Signal] Client connected: ${peerId}`);
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive"
});
const keepAlive = setInterval(() => {
res.write(": keep-alive\n\n");
}, 15e3);
clients.set(peerId, res);
if (messageBuffer.has(peerId)) {
const msgs = messageBuffer.get(peerId);
for (const msg of msgs) {
res.write(`data: ${JSON.stringify(msg)}
`);
}
messageBuffer.delete(peerId);
}
req.on("close", () => {
console.log(`[Signal] Client disconnected: ${peerId}`);
clearInterval(keepAlive);
clients.delete(peerId);
});
return;
}
if (req.method === "POST" && path === "/signal") {
let body = "";
req.on("data", (chunk) => body += chunk);
req.on("end", () => {
try {
const msg = JSON.parse(body);
const target = msg.target;
if (!target) {
res.writeHead(400);
res.end("Missing target");
return;
}
console.log(`[Signal] Relaying ${msg.type} to ${target}`);
if (clients.has(target)) {
clients.get(target).write(`data: ${JSON.stringify(msg)}
`);
} else {
console.log(`[Signal] Target ${target} offline, buffering...`);
if (!messageBuffer.has(target)) {
messageBuffer.set(target, []);
}
messageBuffer.get(target).push(msg);
}
res.writeHead(200);
res.end("Sent");
} catch (e) {
console.error(e);
res.writeHead(500);
res.end(String(e));
}
});
return;
}
res.writeHead(404);
res.end();
});
for (let attempt = 0; attempt < maxTries; attempt++) {
const port = startPort + attempt;
try {
await new Promise((resolve, reject) => {
server.once("error", (err) => {
if (err.code === "EADDRINUSE") {
reject(err);
} else {
reject(err);
}
});
server.listen(port, () => {
server.removeAllListeners("error");
resolve();
});
});
console.log(`[Signal] Server running on port ${port}`);
return {
success: true,
port,
server,
message: `Signaling server started on port ${port}`
};
} catch (err) {
server.removeAllListeners("error");
if (attempt === 0 && autoFindPort) {
console.log(`[Signal] Port ${port} in use, trying to kill existing process...`);
await killProcessOnPort(port);
try {
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, () => {
server.removeAllListeners("error");
resolve();
});
});
console.log(`[Signal] Server running on port ${port} (after kill)`);
return {
success: true,
port,
server,
message: `Signaling server started on port ${port}`
};
} catch {
server.removeAllListeners("error");
}
}
if (!autoFindPort) {
return {
success: false,
error: `Port ${port} is already in use`
};
}
console.log(`[Signal] Port ${port} still in use, trying next...`);
}
}
return {
success: false,
error: `Could not find available port after ${maxTries} attempts starting from ${startPort}`
};
}
// src/ptr/node/network/NetworkSecurity.ts
var NetworkSecurity = class {
config;
constructor(config) {
this.config = config || this.loadSecurityConfigFromEnv();
}
/**
* Load security configuration from environment variables
*/
loadSecurityConfigFromEnv() {
const parseList = (value) => {
if (!value) return void 0;
return value.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
};
return {
allowed_domains: parseList(process.env.CLM_ALLOWED_DOMAINS),
blocked_domains: parseList(process.env.CLM_BLOCKED_DOMAINS),
allowed_protocols: parseList(process.env.CLM_ALLOWED_PROTOCOLS),
block_private_ips: process.env.CLM_BLOCK_PRIVATE_IPS === "true",
block_localhost: process.env.CLM_BLOCK_LOCALHOST === "true"
};
}
/**
* Validate URL against security policy
* Throws SecurityViolationError if URL is not allowed
*/
validateUrl(urlString) {
let url;
try {
url = new URL(urlString);
} catch {
throw this.createSecurityError("DOMAIN_BLOCKED", `Invalid URL: ${urlString}`, urlString);
}
const hostname = url.hostname.toLowerCase();
const protocol = url.protocol.replace(":", "");
if (this.config.blocked_domains) {
for (const pattern of this.config.blocked_domains) {
if (this.matchDomainPattern(hostname, pattern)) {
throw this.createSecurityError(
"DOMAIN_BLOCKED",
`Domain '${hostname}' is blocked by security policy`,
urlString
);
}
}
}
if (this.config.allowed_domains && this.config.allowed_domains.length > 0) {
const isAllowed = this.config.allowed_domains.some(
(pattern) => this.matchDomainPattern(hostname, pattern)
);
if (!isAllowed) {
throw this.createSecurityError(
"DOMAIN_NOT_ALLOWED",
`Domain '${hostname}' is not in the allowed list`,
urlString
);
}
}
if (this.config.allowed_protocols && this.config.allowed_protocols.length > 0) {
if (!this.config.allowed_protocols.includes(protocol)) {
throw this.createSecurityError(
"PROTOCOL_NOT_ALLOWED",
`Protocol '${protocol}' is not allowed. Allowed: ${this.config.allowed_protocols.join(", ")}`,
urlString
);
}
}
if (this.config.block_localhost) {
if (hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") {
throw this.createSecurityError(
"LOCALHOST_BLOCKED",
"Localhost access is blocked by security policy",
urlString
);
}
}
if (this.config.block_private_ips) {
if (this.isPrivateIP(hostname)) {
throw this.createSecurityError(
"PRIVATE_IP_BLOCKED",
`Private IP '${hostname}' is blocked by security policy`,
urlString
);
}
}
}
/**
* Match hostname against domain pattern (supports wildcards like *.example.com)
*/
matchDomainPattern(hostname, pattern) {
const patternLower = pattern.toLowerCase();
if (patternLower.startsWith("*.")) {
const suffix = patternLower.slice(1);
return hostname.endsWith(suffix) || hostname === patternLower.slice(2);
}
return hostname === patternLower;
}
/**
* Check if hostname is a private IP address
*/
isPrivateIP(hostname) {
const privatePatterns = [
/^10\.\d+\.\d+\.\d+$/,
// 10.x.x.x
/^192\.168\.\d+\.\d+$/,
// 192.168.x.x
/^172\.(1[6-9]|2\d|3[01])\.\d+\.\d+$/,
// 172.16-31.x.x
/^169\.254\.\d+\.\d+$/,
// Link-local
/^fc00:/i,
// IPv6 private
/^fd00:/i
// IPv6 private
];
return privatePatterns.some((pattern) => pattern.test(hostname));
}
createSecurityError(code, message, url) {
const error = new Error(message);
error.securityViolation = { code, message, url };
return error;
}
};
// src/ptr/node/network/MCardSerialization.ts
var MCardSerialization = class {
/**
* Serialize an MCard to a JSON-safe payload for network transfer
*/
static serialize(card) {
return {
hash: card.hash,
content: Buffer.from(card.content).toString("base64"),
g_time: card.g_time,
contentType: card.contentType,
hashFunction: card.hashFunction
};
}
/**
* Deserialize a JSON payload back to an MCard
* Uses fromData if hash/g_time provided (preserves identity)
* Otherwise creates new MCard (generates new hash/g_time)
*/
static async deserialize(json) {
if (!json.content) {
throw new Error("Missing content in MCard payload");
}
const content = Buffer.from(json.content, "base64");
if (json.hash && json.g_time) {
return MCard.fromData(content, json.hash, json.g_time);
}
return MCard.create(content);
}
/**
* Verify hash matches content (optional strict mode)
*/
static verifyHash(card, expectedHash) {
if (card.hash !== expectedHash) {
console.warn(`[Network] Hash mismatch. Expected: ${expectedHash}, Got: ${card.hash}`);
return false;
}
return true;
}
};
// src/ptr/node/network/NetworkInfrastructure.ts
var RateLimiter = class {
limits;
defaultLimit;
constructor(tokensPerSecond = 10, maxBurst = 20) {
this.limits = /* @__PURE__ */ new Map();
this.defaultLimit = { tokensPerSecond, maxBurst };
}
/**
* Check if request allowed. Consumes a token if allowed.
*/
check(domain) {
const now = Date.now();
const bucket = this.limits.get(domain) || {
tokens: this.defaultLimit.maxBurst,
lastRefill: now
};
const elapsed = (now - bucket.lastRefill) / 1e3;
const refill = elapsed * this.defaultLimit.tokensPerSecond;
bucket.tokens = Math.min(this.defaultLimit.maxBurst, bucket.tokens + refill);
bucket.lastRefill = now;
if (bucket.tokens >= 1) {
bucket.tokens -= 1;
this.limits.set(domain, bucket);
return true;
}
this.limits.set(domain, bucket);
return false;
}
/**
* Wait until rate limit allows request
*/
async waitFor(domain) {
while (!this.check(domain)) {
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
};
var NetworkCache = class {
memoryCache;
collection;
constructor(collection) {
this.memoryCache = /* @__PURE__ */ new Map();
this.collection = collection;
}
/**
* Generate cache key from request config
*/
static generateKey(method, url, body) {
const keyData = `${method}:${url}:${body || ""}`;
let hash = 0;
for (let i = 0; i < keyData.length; i++) {
const char = keyData.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash;
}
return `cache_${Math.abs(hash).toString(36)}`;
}
/**
* Get cached response if valid
*/
get(cacheKey) {
const cached = this.memoryCache.get(cacheKey);
if (cached && cached.expiresAt > Date.now()) {
return { ...cached.response, timing: { ...cached.response.timing, total: 0 } };
}
if (cached) {
this.memoryCache.delete(cacheKey);
}
return null;
}
/**
* Cache a response with TTL
*/
async set(cacheKey, response, ttlSeconds, persist = false) {
this.memoryCache.set(cacheKey, {
response,
expiresAt: Date.now() + ttlSeconds * 1e3
});
if (persist && this.collection) {
const cacheEntry = {
key: cacheKey,
response,
expiresAt: Date.now() + ttlSeconds * 1e3,
cachedAt: (/* @__PURE__ */ new Date()).toISOString()
};
const card = await MCard.create(JSON.stringify(cacheEntry));
await this.collection.add(card);
}
}
};
var RetryUtils = class {
static calculateBackoffDelay(attempt, strategy, baseDelay, maxDelay) {
let delay;
switch (strategy) {
case "exponential":
delay = baseDelay * Math.pow(2, attempt - 1);
break;
case "linear":
delay = baseDelay * attempt;
break;
case "constant":
default:
delay = baseDelay;
}
const jitter = delay * 0.1 * (Math.random() * 2 - 1);
delay = Math.round(delay + jitter);
return maxDelay ? Math.min(delay, maxDelay) : delay;
}
static shouldRetryStatus(status, retryOn) {
const defaultRetryStatuses = [408, 429, 500, 502, 503, 504];
const retryStatuses = retryOn || defaultRetryStatuses;
return retryStatuses.includes(status);
}
};
// src/ptr/node/network/HttpClient.ts
var HttpClient = class {
rateLimiter;
cache;
constructor(rateLimiter, cache) {
this.rateLimiter = rateLimiter;
this.cache = cache;
}
async request(url, method, headers, body, config) {
const startTime = Date.now();
const fetchUrl = new URL(url);
const cacheConfig = config.cache;
const cacheKey = NetworkCache.generateKey(method, fetchUrl.toString(), typeof body === "string" ? body : void 0);
if (cacheConfig?.enabled && method === "GET") {
const cachedResponse = this.cache.get(cacheKey);
if (cachedResponse) {
console.log(`[Network] Cache hit for ${url}`);
return { ...cachedResponse, cached: true };
}
}
const domain = fetchUrl.hostname;
await this.rateLimiter.waitFor(domain);
const retryConfig = config.retry || {
max_attempts: 1,
backoff: "exponential",
base_delay: 1e3,
max_delay: 3e4
};
let lastError = null;
let lastStatus = null;
let retriesAttempted = 0;
for (let attempt = 1; attempt <= retryConfig.max_attempts; attempt++) {
const timeout = config.timeout || 3e4;
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const ttfbStart = Date.now();
const response = await fetch(fetchUrl.toString(), {
method,
headers,
body,
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok && RetryUtils.shouldRetryStatus(response.status, retryConfig.retry_on)) {
lastStatus = response.status;
if (attempt < retryConfig.max_attempts) {
retriesAttempted++;
const delay = RetryUtils.calculateBackoffDelay(
attempt,
retryConfig.backoff,
retryConfig.base_delay,
retryConfig.max_delay
);
console.log(`[Network] Retry ${attempt}/${retryConfig.max_attempts} for ${url} (status: ${response.status}, delay: ${delay}ms)`);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
}
const ttfbTime = Date.now() - ttfbStart;
let responseBody;
const responseType = config.responseType || "json";
if (responseType === "json") {
try {
responseBody = await response.json();
} catch {
responseBody = await response.text();
}
} else if (responseType === "text") {
responseBody = await response.text();
} else if (responseType === "binary") {
const arrayBuffer = await response.arrayBuffer();
responseBody = Buffer.from(arrayBuffer).toString("base64");
} else {
responseBody = await response.text();
}
const totalTime = Date.now() - startTime;
let mcard_hash;
try {
const bodyStr = typeof responseBody === "string" ? responseBody : JSON.stringify(responseBody);
const responseCard = await MCard.create(bodyStr);
mcard_hash = responseCard.hash;
} catch {
}
const timing = {
dns: 0,
connect: 0,
ttfb: ttfbTime,
total: totalTime
};
const result = {
success: true,
status: response.status,
headers: Object.fromEntries(response.headers.entries()),
body: responseBody,
timing,
mcard_hash
};
if (cacheConfig?.enabled && method === "GET" && response.ok) {
await this.cache.set(cacheKey, result, cacheConfig.ttl, cacheConfig.storage === "mcard");
}
return result;
} catch (error) {
clearTimeout(timeoutId);
lastError = error;
if (attempt < retryConfig.max_attempts) {
retriesAttempted++;
const delay = RetryUtils.calculateBackoffDelay(
attempt,
retryConfig.backoff,
retryConfig.base_delay,
retryConfig.max_delay
);
console.log(`[Network] Retry ${attempt}/${retryConfig.max_attempts} for ${url} (error: ${lastError.message}, delay: ${delay}ms)`);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
}
}
const err = lastError;
return {
success: false,
error: {
code: err?.name === "AbortError" ? "TIMEOUT" : "HTTP_ERROR",
message: err?.message || "Request failed after retries",
status: lastStatus,
retries_attempted: retriesAttempted
}
};
}
};
// src/ptr/node/NetworkRuntime.ts
var NetworkRuntime = class {
collection;
security;
cache;
rateLimiter;
httpClient;
sessions;
constructor(collection) {
this.collection = collection;
this.security = new NetworkSecurity();
this.cache = new NetworkCache(collection);
this.rateLimiter = new RateLimiter();
this.httpClient = new HttpClient(this.rateLimiter, this.cache);
this.sessions = /* @__PURE__ */ new Map();
}
async execute(_code, context, config, _chapterDir) {
const builtin = config.builtin;
if (!builtin) {
throw new Error('NetworkRuntime requires "builtin" to be defined in config.');
}
switch (builtin) {
case "http_request":
return this.handleHttpRequest(config.config || {}, context);
case "http_get":
return this.handleHttpGet(config.config || {}, context);
case "http_post":
return this.handleHttpPost(config.config || {}, context);
case "load_url":
return this.handleLoadUrl(config.config || {}, context);
case "mcard_send":
return this.handleMCardSend(config.config || {}, context);
case "listen_http":
return this.handleListenHttp(config.config || {}, context);
case "mcard_sync":
return this.handleMCardSync(config.config || {}, context);
case "listen_sync":
return this.handleListenSync(config.config || {}, context);
case "webrtc_connect":
return this.handleWebRTCConnect(config.config || {}, context);
case "webrtc_listen":
return this.handleWebRTCListen(config.config || {}, context);
case "session_record":
return this.handleSessionRecord(config.config || {}, context);
case "mcard_read":
return this.handleMCardRead(config.config || {}, context);
case "run_command":
return this.handleRunCommand(config.config, context);
case "clm_orchestrator":
return this.handleOrchestrator(config.config || {}, context);
case "signaling_server":
return this.handleSignalingServer(config.config || {}, context);
default:
throw new Error(`Unknown network builtin: ${builtin}`);
}
}
async handleHttpGet(config, context) {
return this.handleHttpRequest({ ...config, method: "GET" }, context);
}
async handleHttpPost(config, context) {
const params = { ...config, method: "POST" };
if (config.json) {
params.headers = { ...params.headers, "Content-Type": "application/json" };
params.body = JSON.stringify(config.json);
}
return this.handleHttpRequest(params, context);
}
async handleHttpRequest(config, context) {
const url = this.interpolate(config.url, context);
this.security.validateUrl(url);
const method = config.method || "GET";
const headers = this.interpolateHeaders(config.headers || {}, context);
let body = config.body;
if (typeof body === "string") {
body = this.interpolate(body, context);
} else if (typeof body === "object" && body !== null) {
body = JSON.stringify(body);
}
const fetchUrl = new URL(url);
if (config.query_params) {
for (const [key, value] of Object.entries(config.query_params)) {
fetchUrl.searchParams.append(key, this.interpolate(String(value), context));
}
}
return this.httpClient.request(
fetchUrl.toString(),
method,
headers,
body,
{
retry: config.retry,
cache: config.cache,
timeout: typeof config.timeout === "number" ? config.timeout : config.timeout?.total,
responseType: config.response_type
}
);
}
async handleLoadUrl(config, context) {
const url = this.interpolate(config.url, context);
this.security.validateUrl(url);
try {
const res = await fetch(url);
const text = await res.text();
return {
url,
content: text,
status: res.status,
headers: Object.fromEntries(res.headers.entries())
};
} catch (e) {
return { success: false, error: String(e) };
}
}
async handleMCardSend(config, context) {
if (!this.collection) {
throw new Error("MCard Send requires a CardCollection.");
}
const hash = this.interpolate(config.hash, context);
const url = this.interpolate(config.url, context);
const card = await this.collection.get(hash);
if (!card) {
return { success: false, error: `MCard not found: ${hash}` };
}
const payload = MCardSerialization.serialize(card);
return this.handleHttpPost({
url,
json: payload,
headers: config.headers
}, context);
}
async handleListenHttp(config, context) {
const port = Number(this.interpolate(String(config.port || 3e3), context));
const path = this.interpolate(config.path || "/mcard", context);
return new Promise((resolve, reject) => {
const server = http.createServer(async (req, res) => {
if (req.method === "POST" && req.url === path) {
const bodyChunks = [];
req.on("data", (chunk) => bodyChunks.push(chunk));
req.on("end", async () => {
try {
const body = Buffer.concat(bodyChunks).toString();
const json = JSON.parse(body);
const card = await MCardSerialization.deserialize(json);
if (json.hash) {
MCardSerialization.verifyHash(card, json.hash);
}
if (this.collection) {
await this.collection.add(card);
}
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: true, hash: card.hash }));
} catch (e) {
res.writeHead(400, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: false, error: String(e) }));
}
});
} else {
res.writeHead(404);
res.end();
}
});
server.listen(port, () => {
console.log(`[Network] Listening on port ${port} at ${path}`);
resolve({
success: true,
message: `Server started on port ${port}`
});
});
server.on("error", (err) => {
reject(err);
});
});
}
async handleMCardSync(config, context) {
if (!this.collection) {
throw new Error("MCard Sync requires a CardCollection.");
}
const mode = this.interpolate(config.mode || "pull", context);
const urlParams = this.interpolate(config.url, context);
const url = urlParams.endsWith("/") ? urlParams.slice(0, -1) : urlParams;
const localCards = await this.collection.getAllMCardsRaw();
const localHashes = new Set(localCards.map((c) => c.hash));
const manifestRes = await this.handleHttpRequest({
url: `${url}/manifest`,
method: "GET"
}, context);
if (!manifestRes.success) {
throw new Error(`Failed to fetch remote manifest: ${manifestRes.error?.message}`);
}
const remoteHashes = new Set(manifestRes.body);
const stats = {
mode,
local_total: localHashes.size,
remote_total: remoteHashes.size,
synced: 0,
pushed: 0,
pulled: 0
};
const pushCards = async () => {
const toSend = [];
for (const card of localCards) {
if (!remoteHashes.has(card.hash)) {
toSend.push(card);
}
}
if (toSend.length > 0) {
const payload = {
cards: toSend.map((card) => MCardSerialization.serialize(card))
};
const pushRes = await this.handleHttpPost({
url: `${url}/batch`,
json: payload,
headers: config.headers
}, context);
if (!pushRes.success) {
throw new Error(`Failed to push batch: ${pushRes.error?.message}`);
}
return toSend.length;
}
return 0;
};
const pullCards = async () => {
const neededHashes = [];
for (const h of remoteHashes) {
if (!localHashes.has(h)) {
neededHashes.push(h);
}
}
if (neededHashes.length > 0) {
const fetchRes = await this.handleHttpPost({
url: `${url}/get`,
json: { hashes: neededHashes },
headers: config.headers
}, context);
if (!fetchRes.success) {
throw new Error(`Failed to pull batch: ${fetchRes.error?.message}`);
}
const receivedCards = fetchRes.body.cards;
for (const json of receivedCards) {
const card = await MCardSerialization.deserialize(json);
await this.collection.add(card);
}
return receivedCards.length;
}
return 0;
};
if (mode === "push") {
stats.pushed = await pushCards();
stats.synced = stats.pushed;
} else if (mode === "pull") {
stats.pulled = await pullCards();
stats.synced = stats.pulled;
} else if (mode === "both" || mode === "bidirectional") {
const pushed = await pushCards();
const pulled = await pullCards();
stats.synced = pushed + pulled;
stats.pushed = pushed;
stats.pulled = pulled;
}
return { success: true, stats };
}
// ============ WebRTC Implementation ============
getPeerConnectionClass() {
if (typeof RTCPeerConnection !== "undefined") {
return RTCPeerConnection;
} else if (typeof global !== "undefined" && global.RTCPeerConnection) {
return global.RTCPeerConnection;
}
return null;
}
async handleWebRTCConnect(config, context) {
const PeerConnection = this.getPeerConnectionClass();
if (!PeerConnection) {
return {
success: false,
error: "WebRTC not supported in this environment (RTCPeerConnection not found)."
};
}
const signalingUrl = this.interpolate(config.signaling_url, context);
const targetPeerId = this.interpolate(config.target_peer_id, context);
const myPeerId = config.peer_id ? this.interpolate(config.peer_id, context) : `peer_${Date.now()}`;
const channelLabel = config.channel_label || "mcard-sync";
if (signalingUrl === "mock://p2p") {
return new Promise((resolve) => {
setTimeout(() => {
resolve({
success: true,
peer_id: myPeerId,
channel: channelLabel,
status: "connected",
mock: true
});
}, 100);
});
}
console.log(`[WebRTC] Connecting to ${targetPeerId} via ${signalingUrl} as ${myPeerId}`);
const pc = new PeerConnection({
iceServers: config.ice_servers || [{ urls: "stun:stun.l.google.com:19302" }]
});
const dc = pc.createDataChannel(channelLabel);
const connectionPromise = new Promise((resolve, reject) => {
const timeoutMs = config.timeout || 3e4;
const timeoutId = setTimeout(() => {
pc.close();
reject(new Error("WebRTC connection timed out"));
}, timeoutMs);
dc.onopen = () => {
clearTimeout(timeoutId);
console.log(`[WebRTC] Data channel '${channelLabel}' open`);
if (config.message) {
const msg = typeof config.message === "string" ? this.interpolate(config.message, context) : JSON.stringify(config.message);
dc.send(msg);
}
resolve({
success: true,
peer_id: myPeerId,
channel: channelLabel,
status: "connected"
});
};
dc.onerror = (err) => {
clearTimeout(timeoutId);
console.error("[WebRTC] Data channel error:", err);
reject(err);
};
this._setupP2PProtocol(dc);
});
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
console.log("[WebRTC] Local Offer created. SDP ready to send.");
if (config.await_response !== false) {
return connectionPromise;
}
return {
success: true,
status: "initiating",
peer_id: myPeerId
};
}
_setupP2PProtocol(dc) {
dc.onmessage = async (event) => {
try {
const msg = JSON.parse(event.data);
if (msg.type === "sync_manifest") {
if (!this.collection) return;
const remoteHashes = new Set(msg.hashes);
const localCards = await this.collection.getAllMCardsRaw();
const localHashes = new Set(localCards.map((c) => c.hash));
const needed = [...remoteHashes].filter((h) => !localHashes.has(h));
const toPush = localCards.filter((c) => !remoteHashes.has(c.hash));
if (needed.length > 0) {
dc.send(JSON.stringify({ type: "sync_request", hashes: needed }));
}
if (toPush.length > 0) {
const payload = {
type: "batch_push",
cards: toPush.map((c) => MCardSerialization.serialize(c))
};
dc.send(JSON.stringify(payload));
}
} else if (msg.type === "sync_request") {
if (!this.collection) return;
const requested = msg.hashes || [];
const foundCards = [];
for (const h of requested) {
const c = await this.collection.get(h);
if (c) foundCards.push(MCardSerialization.serialize(c));
}
if (foundCards.length > 0) {
dc.send(JSON.stringify({ type: "batch_push", cards: foundCards }));
}
} else if (msg.type === "batch_push") {
if (!this.collection) return;
const cards = msg.cards || [];
let added = 0;
for (const cJson of cards) {
const card = await MCardSerialization.deserialize(cJson);
await this.collection.add(card);
added++;
}
console.log(`[WebRTC] Synced ${added} cards from peer.`);
}
} catch (e) {
console.error("[WebRTC] Protocol error:", e);
}
};
}
async handleWebRTCListen(config, context) {
const PeerConnection = this.getPeerConnectionClass();
if (!PeerConnection) {
return {
success: false,
error: "WebRTC not supported in this environment (RTCPeerConnection not found)."
};
}
const signalingUrl = this.interpolate(config.signaling_url, context);
const myPeerId = config.peer_id ? this.interpolate(config.peer_id, context) : `listener_${Date.now()}`;
if (signalingUrl === "mock://p2p") {
return new Promise((resolve) => {
setTimeout(() => {
resolve({
success: true,
peer_id: myPeerId,
status: "listening",
mock: true
});
}, 100);
});
}
console.log(`[WebRTC] Listening on ${signalingUrl} as ${myPeerId}`);
return {
success: true,
status: "listening",
peer_id: myPeerId,
note: "Signaling loop implementation pending specific server protocol."
};
}
async handleListenSync(config, context) {
if (!this.collection) {
throw new Error("Listen Sync requires a CardCollection.");
}
const port = Number(this.interpolate(String(config.port || 3e3), context));
const basePath = this.interpolate(config.base_path || "/sync", context);
return new Promise((resolve, reject) => {
const server = http.createServer(async (req, res) => {
const url = req.url || "";
const readBody = async () => {
return new Promise((res2, rej) => {
const chunks = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
try {
const str = Buffer.concat(chunks).toString();
res2(JSON.parse(str || "{}"));
} catch (e) {
rej(e);
}
});
req.on("error", rej);
});
};
try {
if (req.method === "GET" && url === `${basePath}/manifest`) {
const all = await this.collection.getAllMCardsRaw();
const hashes = all.map((c) => c.hash);
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify(hashes));
return;
}
if (req.method === "POST" && url === `${basePath}/batch`) {
const json = await readBody();
const cards = json.cards || [];
let added = 0;
for (const cJson of cards) {
const card = await MCardSerialization.deserialize(cJson);
await this.collection.add(card);
added++;
}
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: true, added }));
return;
}
if (req.method === "POST" && url === `${basePath}/get`) {
const json = await readBody();
const requestedHashes = json.hashes || [];
const foundCards = [];
for (const h of requestedHashes) {
const card = await this.collection.get(h);
if (card) {
foundCards.push(MCardSerialization.serialize(card));
}
}
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: true, cards: foundCards }));
return;
}
res.writeHead(404);
res.end();
} catch (e) {
res.writeHead(500, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: false, error: String(e) }));
}
});
server.listen(port, () => {
console.log(`[Network] Sync listening on port ${port} at ${basePath}`);
resolve({
success: true,
message: `Sync Server started on port ${port}`,
port,
basePath
});
});
server.on("error", (err) => {
reject(err);
});
});
}
interpolate(text, context) {
if (!text || typeof text !== "string") return text;
return text.replace(/\$\{([^}]+)\}/g, (_, path) => {
const keys = path.split(".");
let val = context;
for (const key of keys) {
if (val && typeof val === "object" && key in val) {
val = val[key];
} else {
return "";
}
}
return String(val);
});
}
interpolateHeaders(headers, context) {
const result = {};
for (const [key, val] of Object.entries(headers)) {
result[key] = this.interpolate(val, context);
}
return result;
}
async handleSessionRecord(config, context) {
if (!this.collection) {
throw new Error("Session Record requires a CardCollection.");
}
const sessionId = this.interpolate(config.sessionId, context);
let operation = config.operation || "add";
if (typeof operation === "string" && operation.includes("${")) {
operation = this.interpolate(operation, context);
}
if (operation === "init") {
if (this.sessions.has(sessionId)) {
return { success: true, message: "Session already exists", sessionId };
}
let bufferSize = config.maxBufferSize || 5;
if (typeof config.maxBufferSize === "string") {
bufferSize = parseInt(this.interpolate(config.maxBufferSize, context), 10);
}
let initialHead = config.initialHeadHash || null;
if (typeof config.initialHeadHash === "string") {
initialHead = this.interpolate(config.initialHeadHash, context);
if (initialHead === "null" || initialHead === "undefined" || initialHead === "") initialHead = null;
}
const session2 = new P2PChatSession(this.collection, sessionId, bufferSize, initialHead);
this.sessions.set(sessionId, session2);
return { success: true, message: "Session initialized", sessionId, bufferSize, initialHead };
}
if (operation === "batch") {
const results = [];
let subOps = config.operations;
if (!Array.isArray(subOps)) {
const ctx = context;
subOps = ctx?.params?.operations || ctx?.operations || [];
}
for (const op of subOps) {
const subConfig = { ...config, ...op };
results.push(await this.handleSessionRecord(subConfig, context));
}
return {
success: true,
operation: "batch",
results
};
}
if (operation === "summarize") {
let session2 = this.sessions.get(sessionId);
if (!session2) {
session2 = new P2PChatSession(this.collection, sessionId, 5, null);
this.sessions.set(sessionId, session2);
}
const keepOriginals = config.keepOriginals === true;
const summaryHash = await session2.summarize(keepOriginals);
return {
success: true,
operation: "summarize",
summary_hash: summaryHash,
sessionId
};
}
const session = this.sessions.get(sessionId);
if (!session) {
const newSession = new P2PChatSession(this.collection, sessionId, 5, null);
this.sessions.set(sessionId, newSession);
}
const validSession = this.sessions.get(sessionId);
if (operation === "add") {
const sender = this.interpolate(config.sender || "unknown", context);
const content = this.interpolate(config.content || "", context);
const hash = await validSession.addMessage(sender, content);
const head = validSession.getHeadHash();
return {
success: true,
checkpoint_hash: hash,
head_hash: head,
sessionId
};
} else if (operation === "flush") {
const hash = await validSession.checkpoint();
return {
success: true,
checkpoint_hash: hash,
sessionId
};
}
return { success: false, error: `Unknown operation ${operation}` };
}
async handleMCardRead(config, context) {
if (!this.collection) {
throw new Error("MCard Read requires a CardCollection.");
}
const hash = this.interpolate(config.hash, context);
if (!hash) throw new Error("Hash is required for mcard_read");
const card = await this.collection.get(hash);
if (!card) return { success: false, error: "MCard not found", hash };
let content = card.getContentAsText();
if (config.parse_json !== false) {
try {
content = JSON.parse(content);
} catch (e) {
}
}
return {
success: true,
hash,
content,
g_time: card.g_time
};
}
async handleOrchestrator(config, context) {
const steps = config.steps || [];
const state = {};
let allSuccess = true;
console.log(`[NetworkRuntime] Starting Orchestration with ${steps.length} steps.`);
for (const step of steps) {
const stepName = step.name || step.action;
console.log(`[Orchestrator] Step: ${stepName}`);
try {
if (step.action === "start_process") {
const cmd = this.interpolate(step.command, context);
const { spawn } = await import("child_process");
const parts = cmd.split(" ");
const env = { ...process.env, ...step.env || {} };
const proc = spawn(parts[0], parts.slice(1), {
detached: true,
stdio: "inherit",
cwd: process.cwd(),
env
});
proc.unref();
if (step.id_key) {
state[step.id_key] = proc.pid;
console.log(`[Orchestrator] Process started (PID: ${proc.pid}) stored in '${step.id_key}'`);
} else {
console.log(`[Orchestrator] Process started (PID: ${proc.pid})`);
}
if (step.wait_after) {
await new Promise((r) => setTimeout(r, step.wait_after));
}
} else if (step.action === "run_clm") {
if (!context.runCLM) throw new Error("runCLM capability not available in context");
const file = step.file;
const input = step.input || {};
console.log(`[Orchestrator] Running CLM: ${file}`);
const res = await context.runCLM(file, input);
if (!res.success) {
console.error(`[Orchestrator] CLM Failed: ${file}`, res.error);
if (!step.continue_on_error) {
allSuccess = false;
break;
}
} else {
console.log(`[Orchestrator] CLM Passed: ${file}`);
}
} else if (step.action === "run_clm_background") {
const file = step.file;
const filter = file.replace(/\.(yaml|yml|clm)$/i, "");
const cmd = `npx tsx examples/run-all-clms.ts ${filter}`;
const { spawn } = await import("child_process");
const parts = cmd.split(" ");
const env = { ...process.env, ...step.env || {} };
const proc = spawn(parts[0], parts.slice(1), {
detached: true,
stdio: "inherit",
cwd: process.cwd(),
env
});
proc.unref();
if (step.id_key) {
state[step.id_key] = proc.pid;
console.log(`[Orchestrator] Background CLM started (PID: ${proc.pid}) stored in '${step.id_key}'`);
}
if (step.wait_after) {
await new Promise((r) => setTimeout(r, step.wait_after));
}
} else if (step.action === "stop_process") {
const key = step.pid_key;
const pid = state[key];
if (pid) {
try {
context.process.kill(pid);
console.log(`[Orchestrator] Stopped process ${pid} (${key})`);
} catch (e) {
console.warn(`[Orchestrator] Failed to stop process ${pid}: ${e}`);
}
} else {
console.warn(`[Orchestrator] No PID found for key '${key}'`);
}
} else if (step.action === "sleep") {
const ms = step.ms || 1e3;
await new Promise((r) => setTimeout(r, ms));
} else if (step.action === "start_signaling_server") {
const port = step.port || 3e3;
console.log(`[Orchestrator] Starting builtin signaling server on port ${port}...`);
const result = await this.handleSignalingServer({ port, background: true }, context);
if (result.success) {
console.log(`[Orchestrator] Signaling server started on port ${result.port}`);
if (step.id_key) {
state[step.id_key] = {
type: "signaling_server",
port: result.port,
server: this._signalingServer
};
}
} else {
console.error(`[Orchestrator] Failed to start signaling server: ${result.error}`);
if (!step.continue_on_error) {
allSuccess = false;
break;
}
}
if (step.wait_after) {
await new Promise((r) => setTimeout(r, step.wait_after));
}
} else if (step.action === "stop_signaling_server") {
const key = step.id_key;
const serverInfo = state[key];
if (serverInfo && serverInfo.server) {
try {
serverInfo.server.close();
console.log(`[Orchestrator] Signaling server stopped (${key})`);
} catch (e) {
console.warn(`[Orchestrator] Failed to stop signaling server: ${e}`);
}