@sema-agent/core
Version:
Stateless, task-oriented AI agent core
393 lines (392 loc) • 20.9 kB
JavaScript
import { createHash } from "node:crypto";
import { types } from "node:util";
import { CREDENTIAL_MIN_CHARS, CREDENTIAL_VOCABULARY, envNameView, isSensitiveHeaderName, splitCamelCase } from "./credential-vocabulary.js";
import { DEFAULT_INHERITED_ENV_VARS } from "@modelcontextprotocol/sdk/client/stdio.js";
import { deliverEngineNotice } from "./engine-notice.js";
import { receiveOwnProperties } from "./own-property-receipt.js";
const GIT_CONFIG_ENV_RE = /^GIT_CONFIG_(?:COUNT|PARAMETERS|(?:KEY|VALUE)_[0-9]+)$/;
const PROXY_ENV_RE = /^(?:(?:https?|ftp|all|no)_proxy|npm_config_(?:https?_)?proxy|npm_config_noproxy|yarn_proxy|(?:yarn|global_agent|docker|claude_code)_(?:https?|no)_proxy|cloudsdk_proxy_[a-z]+|electron_get_use_proxy)$/i;
function isExemptEnvName(name) {
return GIT_CONFIG_ENV_RE.test(name) || PROXY_ENV_RE.test(name);
}
const ENV_NAMES = envNameView(CREDENTIAL_VOCABULARY);
const OTEL_ENV_RE = /^(?:INPUT_)?OTEL_/i;
const SEMA_NAMESPACE_SECRET_RE = new RegExp(`^SEMA_(?:.*_)?${ENV_NAMES.segmentWord}(?:_|$)`, "is");
const BUNDLER_SETTINGS = ["BUILD", "LOCAL", "MIRROR", "PATH", "WITH", "WITHOUT", "CACHE", "DISABLE", "IGNORE", "ONLY"];
const BUNDLER_HOST = "(?:[A-Za-z0-9]+(?:___[A-Za-z0-9]+)*__)+[A-Za-z]{2,}";
const BUNDLER_CREDENTIAL_RE = new RegExp(`^(?:INPUT_)?BUNDLE_(?!(?:${BUNDLER_SETTINGS.join("|")})__(?!${BUNDLER_HOST}$))\\w*__`, "i");
const PUBLISHER_PREFIX_RE = /^(?:INPUT_|ORG_GRADLE_PROJECT_|POETRY_PYPI_TOKEN_|POETRY_HTTP_BASIC_|CARGO_REGISTRIES_|CONAN_LOGIN_USERNAME_|CONAN_PASSWORD_)/i;
const PUBLISHER_LOGIN_PREFIX_RE = /^(?:INPUT_)?(?:POETRY_HTTP_BASIC_|CONAN_LOGIN_USERNAME_)/i;
const USER_NAME_TAIL_RE = /USER(?:_?NAME)?_?[0-9]*$/i;
function nameShapeHolds(name) {
const words = name.replace(ENV_NAMES.nonCredentialPrefix, "").replace(/-/g, "_");
const split = splitCamelCase(words);
const { credentialWord, connectionStringName } = ENV_NAMES;
const shaped = credentialWord.test(words) || credentialWord.test(split) || connectionStringName.test(words) || connectionStringName.test(split) || BUNDLER_CREDENTIAL_RE.test(words);
return shaped && !ENV_NAMES.nonSecretNameShape.test(name);
}
function nameIsCredentialShaped(name) {
if (BUNDLER_CREDENTIAL_RE.test(name) || nameShapeHolds(name))
return true;
const words = name.replace(/-/g, "_");
return PUBLISHER_PREFIX_RE.test(words) && (PUBLISHER_LOGIN_PREFIX_RE.test(words) || USER_NAME_TAIL_RE.test(words) || nameShapeHolds(words));
}
const VALUE_SCAN_LIMIT = 8192;
function looksLikeToken(user) {
return ENV_NAMES.tokenPrefix.test(user) || (user.length >= 20 && /[0-9]/.test(user) && /[a-z]/i.test(user));
}
const HOST_RE = /^(?:localhost|[a-z0-9_-]+(?:\.[a-z0-9_-]+)*\.(?=[a-z0-9-]*[a-z])[a-z0-9-]+|[a-z0-9_.-]+(?=:\d+(?:[/?#]|$))|\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-f:.]+\])(?::\d+)?(?:[/?#]|$)/i;
const HOSTLIKE_RE = /^(?:(?=[a-z0-9._-]*[a-z])[a-z0-9._-]+|\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-f:.]+\])(?::\d+)?(?:[/?#]|$)/i;
function isHostPortPathBeforeAt(before, after) {
const hostPort = /^([a-z0-9_-]+(?:\.[a-z0-9_-]+)*):\d+([/?#].*)$/is.exec(before);
if (!hostPort)
return false;
if (/[?#&]/.test(hostPort[2] ?? ""))
return true;
if (HOST_RE.test(after))
return false;
if (before.endsWith("/"))
return true;
if (/^(?:v?\d|sha\d*:)/i.test(after))
return true;
return !HOSTLIKE_RE.test(after);
}
function authorityCarriesCredential(rest) {
const blank = rest.search(/\s/);
const word = blank === -1 ? rest : rest.slice(0, blank);
const at = word.lastIndexOf("@");
if (at === -1)
return false;
const userinfo = word.slice(0, at);
if (isHostPortPathBeforeAt(userinfo, word.slice(at + 1)))
return false;
return userinfo.includes(":") || (!/[/?#]/.test(userinfo) && looksLikeToken(userinfo));
}
const PACKAGE_REFERENCE_RE = /^(?:[\w.-]+(?::[\w.-]+)?@sha(?:256|384|512):[0-9a-f]{32,}|npm:(?:@[\w.-]+\/)?[\w.-]+@[\w.^~<>=*|+-]*|[a-z_][\w.-]*:[\w.-]+:v?\d[\w.+\[\](),-]*@(?:jar|war|ear|aar|apk|aab|pom|zip|tar|tgz|module|klib|exe|dll|so|dylib)?)$/i;
function carriesSchemelessUserinfo(text) {
if (!/^[^\s/@:]+:(?!\/\/)(?![\\/])[^\s@]*@[^\s/@]+/.test(text) || /^[a-z]:[\\/]/i.test(text) || PACKAGE_REFERENCE_RE.test(text))
return false;
const at = text.lastIndexOf("@", text.search(/\s|$/));
return !isHostPortPathBeforeAt(text.slice(0, at), text.slice(at + 1)) && !/^(?:mailto|sips?|xmpp|im|acct):/i.test(text);
}
const FLAG_WORDS = ["true", "false", "none", "null", "yes", "no", "on", "off", "enabled", "disabled", "required", "optional"];
const CREDENTIAL_ASSIGNMENT_RE = new RegExp(String.raw `(?:(?:^|[;&?#,{]|\/:)\s*|\s)["']?(?!-*jobserver-auth\s*[=:])(?:[a-z0-9_.-]{0,64}(?:password|passwd|pwd|secret|token|(?:account|access|api|private|subscription)[-_]?key|signature|sig|credential)|[a-z0-9_.-]{0,63}[_.-]auth)["']?\s*[=:]\s*["']?(?!(?:${FLAG_WORDS.join("|")})(?:$|[;&,\s"']))[^;&,\s"']+`, "i");
const WEBHOOK_URL_RE = /https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/)[\w\/@.~-]{16,}/i;
const SONAR_LOGIN_RE = new RegExp(String.raw `["']?sonar\.login["']?\s*[:=]\s*["']?[^\s"',}]{${CREDENTIAL_MIN_CHARS},}`);
const PRIVATE_KEY_HEADER_RE = /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/;
const AUTHORIZATION_HEADER_RE = new RegExp(String.raw `\bauthorization\s*[:=]\s*["']?[a-z][a-z0-9_-]*\s+[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`, "i");
const TOKEN_WORD_RE = /\b(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=[A-Za-z0-9_~+=-]*[A-Z])(?=([A-Za-z0-9_~+=-]{20,}))\1(?![\/.])/;
const BEARER_BASIC_RE = new RegExp(String.raw `\b(?:Bearer|Basic)\s+(?=[A-Za-z._~+\/=-]*[0-9]|(?:[A-Za-z0-9._~+\/=-]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9.-]*[_~+\/=])[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`);
export function looksLikeCredentialValue(value) {
const text = value.length > VALUE_SCAN_LIMIT ? value.slice(0, VALUE_SCAN_LIMIT) : value;
for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", at + 3)) {
if (at > 0 && /[a-z0-9+.-]/i.test(text[at - 1] ?? "") && authorityCarriesCredential(text.slice(at + 3)))
return true;
}
if (text.startsWith("//") && authorityCarriesCredential(text.slice(2)))
return true;
return (carriesSchemelessUserinfo(text) ||
CREDENTIAL_ASSIGNMENT_RE.test(text) ||
WEBHOOK_URL_RE.test(text) ||
SONAR_LOGIN_RE.test(text) ||
PRIVATE_KEY_HEADER_RE.test(text) ||
AUTHORIZATION_HEADER_RE.test(text) ||
TOKEN_WORD_RE.test(text) ||
BEARER_BASIC_RE.test(text));
}
const SECRET_ENV_KIND_CONFIDENCE = {
"exact-name": "high",
"prefix-rule": "high",
"value-rule": "high",
"own-credential": "high",
"name-rule": "medium",
"namespace-rule": "medium",
};
function sensitiveNameKind(name) {
if (ENV_NAMES.exactNames.has(name.toUpperCase().replace(/-/g, "_")))
return "exact-name";
if (OTEL_ENV_RE.test(name))
return "prefix-rule";
if (SEMA_NAMESPACE_SECRET_RE.test(name))
return "namespace-rule";
return !ENV_NAMES.nonSecretExactNames.has(name.toUpperCase()) && nameIsCredentialShaped(name) ? "name-rule" : undefined;
}
export function isSensitiveName(key) {
return sensitiveNameKind(key) !== undefined;
}
function classifySecretEnvEntry(name, value) {
if (isExemptEnvName(name))
return undefined;
const named = sensitiveNameKind(name);
if (named !== undefined)
return named;
return value !== undefined && looksLikeCredentialValue(typeof value === "string" ? value : String(value)) ? "value-rule" : undefined;
}
export function isSecretEnvKey(key, value) {
return classifySecretEnvEntry(key, value) !== undefined;
}
export function scrubSecretEnv(env, findings) {
const kept = [];
for (const key of Object.keys(env)) {
const value = env[key];
const kind = classifySecretEnvEntry(key, value);
if (kind === undefined)
kept.push([key, value]);
else
findings?.push({ key, kind, confidence: SECRET_ENV_KIND_CONFIDENCE[kind], source: "secret-env" });
}
return Object.fromEntries(kept);
}
function envNameKey(name) {
return process.platform === "win32" ? name.toUpperCase() : name;
}
const UNPAIRED_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
function variableNameListProblem(label, value) {
const bad = value.findIndex((k) => typeof k !== "string" || k.length === 0 || /[=\0\s]/.test(k) || UNPAIRED_SURROGATE_RE.test(k));
if (bad === -1)
return undefined;
const k = value[bad];
const why = typeof k !== "string" ? typeof k : k === "" ? "an empty string" : `a name holding ${k.includes("=") ? "\"=\"" : k.includes("\0") ? "a NUL" : /\s/.test(k) ? "whitespace" : "an unpaired surrogate"}`;
return `${label}[${bad}] is not a variable name (got ${why})`;
}
const WINDOWS_REQUIRED_ENV = Object.freeze(["HOMEDRIVE", "HOMEPATH", "LOGONSERVER", "PATH", "SYSTEMDRIVE", "SYSTEMROOT", "TEMP", "USERDOMAIN", "USERNAME", "USERPROFILE", "WINDIR"]);
function runtimeSuppliedEnv() {
return ["NODE_V8_COVERAGE", ...DEFAULT_INHERITED_ENV_VARS, ...(process.platform === "win32" ? WINDOWS_REQUIRED_ENV : [])];
}
const valueDigest = (value) => createHash("sha256").update(value, "utf16le").digest("base64");
const indexHas = (index, value) => {
const digests = index.get(value.length);
return digests !== undefined && digests.has(valueDigest(value));
};
const indexAdd = (index, value) => {
const digests = index.get(value.length);
if (digests === undefined)
index.set(value.length, new Set([valueDigest(value)]));
else
digests.add(valueDigest(value));
};
const WITHHELD_ENV = Symbol.for("sema.core.withheldEnvTable");
const NOTHING_NAMED = Object.freeze([]);
const isWithheldEnvTable = (t) => typeof t === "object" && t !== null && Array.isArray(t.names) && t.declared instanceof Map && t.held instanceof Map;
const withheldEnvOf = () => {
const slot = Reflect.get(globalThis, WITHHELD_ENV);
if (isWithheldEnvTable(slot))
return slot;
const fresh = { names: NOTHING_NAMED, declared: new Map(), held: new Map() };
Reflect.set(globalThis, WITHHELD_ENV, fresh);
return fresh;
};
const kindOf = (v) => (v === null ? "null" : Array.isArray(v) ? "an array" : typeof v);
function listCopy(label, list) {
if (types.isProxy(list))
throw new Error(`declareWithheldEnv: ${label} must be a plain array, not a Proxy`);
const copy = [];
try {
for (let i = 0, n = list.length; i < n; i++)
copy[i] = list[i];
}
catch {
throw new Error(`declareWithheldEnv: an element of ${label} threw while it was read — refused`);
}
return Object.freeze(copy);
}
const CANNOT_BE_WITHHELD = "which a child is handed from this process's environment whatever the spawn passes (by Node, the MCP stdio transport or Windows) — it cannot be withheld";
function runtimeSuppliedHolderOf(value) {
const supplied = policyArm([...runtimeSuppliedEnv()]);
return Object.keys(supplied).find((k) => supplied[k] === value);
}
const PLACEHOLDER_KEY_WORDS = new Set([...FLAG_WORDS, "undefined", "placeholder", "lm-studio", "not-needed", "sk-no-key-required"]);
const ONE_REPEATED_CHARACTER_RE = /^(\S)\1{2,}$/u;
function notACredentialReason(value) {
const text = value.trim();
if (text === "")
return "white space only, which every blank variable equals";
if (text.length < CREDENTIAL_MIN_CHARS)
return `a string of fewer than ${CREDENTIAL_MIN_CHARS} characters, which names no secret`;
if (PLACEHOLDER_KEY_WORDS.has(text.toLowerCase()))
return "a flag or placeholder word, which names no secret";
if (ONE_REPEATED_CHARACTER_RE.test(text.replace(CREDENTIAL_VOCABULARY.tokenPrefix, "")))
return "one character repeated, which names no secret";
return undefined;
}
function credentialValueProblem(v) {
if (typeof v !== "string")
return kindOf(v);
if (v === "")
return "an empty string, which every variable set to nothing equals";
if (v.includes("\0") || UNPAIRED_SURROGATE_RE.test(v))
return `a value holding ${v.includes("\0") ? "a NUL" : "an unpaired surrogate"}, which no variable can hold`;
return notACredentialReason(v);
}
function credentialValueListProblem(value) {
for (const [i, v] of value.entries()) {
const why = credentialValueProblem(v);
if (why !== undefined)
return `values[${i}] is not a credential value (got ${why})`;
}
return undefined;
}
export function declareWithheldEnv(table) {
const given = table;
if (given === null || typeof given !== "object" || Array.isArray(given)) {
throw new Error(`declareWithheldEnv: the declaration must be a table { names, values } (got ${Array.isArray(given) ? "an array — a names-only list is no longer a declaration: pass { names, values }" : kindOf(given)})`);
}
const received = receiveOwnProperties(given, { subject: "declareWithheldEnv's table", unit: "member", moment: "at the declaration" });
const stray = Object.keys(received).find((k) => k !== "names" && k !== "values");
if (stray !== undefined)
throw new Error(`declareWithheldEnv: the table has a member ${JSON.stringify(stray.slice(0, 64))} — it holds names and values only`);
const givenNames = received.names;
if (!Array.isArray(givenNames))
throw new Error(`declareWithheldEnv: names must be an array of variable names (got ${kindOf(givenNames)})`);
const names = listCopy("names", givenNames);
const problem = variableNameListProblem("names", names);
if (problem !== undefined)
throw new Error(`declareWithheldEnv: ${problem}`);
const supplied = new Set(runtimeSuppliedEnv().map(envNameKey));
const unwithholdable = names.findIndex((k) => supplied.has(envNameKey(k)));
if (unwithholdable !== -1)
throw new Error(`declareWithheldEnv: names[${unwithholdable}] is ${String(names[unwithholdable])}, ${CANNOT_BE_WITHHELD}`);
const givenValues = received.values;
if (!Array.isArray(givenValues))
throw new Error(`declareWithheldEnv: values must be an array of credential values (got ${kindOf(givenValues)})`);
const values = listCopy("values", givenValues);
const valueProblem = credentialValueListProblem(values);
if (valueProblem !== undefined)
throw new Error(`declareWithheldEnv: ${valueProblem}`);
for (const [i, v] of values.entries()) {
const holder = runtimeSuppliedHolderOf(v);
if (holder !== undefined)
throw new Error(`declareWithheldEnv: values[${i}] is the value of ${holder}, ${CANNOT_BE_WITHHELD}`);
}
const declared = new Map();
for (const v of values)
indexAdd(declared, v);
Reflect.set(globalThis, WITHHELD_ENV, { names: names, declared, held: withheldEnvOf().held });
}
export function holdOwnCredentialValues(values, onNotice) {
const given = [...values].filter((v) => typeof v === "string" && v.length > 0);
const taken = given.filter((v) => {
const why = notACredentialReason(v);
if (why !== undefined)
sayNotTaken(v, why, onNotice);
return why === undefined;
});
for (const v of taken) {
const holder = runtimeSuppliedHolderOf(v);
if (holder !== undefined)
throw Object.assign(new Error(`a credential the engine was handed is the value of ${holder}, ${CANNOT_BE_WITHHELD}; the credential is refused`), { code: "config.invalid_argument" });
}
const { held } = withheldEnvOf();
for (const v of taken)
indexAdd(held, v);
}
const AUTH_SCHEME_VALUE_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+ +(\S[\s\S]*)$/;
const SCHEMED_HEADER_RE = /^(?:proxy-)?authorization$/i;
export function isCredentialHeaderName(name) {
return isSensitiveHeaderName(name);
}
export function isSchemedCredentialHeader(name) {
return SCHEMED_HEADER_RE.test(name);
}
export function credentialHeaderValues(headers) {
const out = [];
for (const [name, value] of Object.entries(headers ?? {})) {
if (typeof value !== "string" || !(isCredentialHeaderName(name) || looksLikeCredentialValue(value)))
continue;
const credentials = AUTH_SCHEME_VALUE_RE.exec(value)?.[1];
if (credentials !== undefined && isSchemedCredentialHeader(name) && notACredentialReason(credentials) !== undefined) {
out.push(credentials);
continue;
}
out.push(value);
if (credentials !== undefined)
out.push(credentials);
}
return out;
}
const intakeSaid = new Set();
function sayNotTaken(value, why, onNotice) {
const key = valueDigest(value.trim());
if (intakeSaid.has(key))
return;
intakeSaid.add(key);
try {
deliverEngineNotice(onNotice, {
code: "config.credential_not_withheld",
message: `a credential the engine was handed is ${why} — it is used as given, but a variable holding the same string is not withheld from a spawned child (said once per value; the value is not repeated)`,
detail: { reason: why },
});
}
catch {
}
}
export function __resetWithheldEnv() {
Reflect.set(globalThis, WITHHELD_ENV, { names: NOTHING_NAMED, declared: new Map(), held: new Map() });
intakeSaid.clear();
}
function withholdOwnCredentials(inherited, findings) {
const table = withheldEnvOf();
if (table.names.length === 0 && table.declared.size === 0 && table.held.size === 0)
return inherited;
const own = new Set(table.names.map(envNameKey));
const kept = [];
for (const key of Object.keys(inherited)) {
const value = inherited[key];
const ownValue = typeof value === "string" && (indexHas(table.declared, value) || indexHas(table.held, value));
if (!own.has(envNameKey(key)) && !ownValue)
kept.push([key, value]);
else
findings?.push({ key, kind: "own-credential", confidence: SECRET_ENV_KIND_CONFIDENCE["own-credential"], source: "secret-env" });
}
return Object.fromEntries(kept);
}
function policyShapeProblem(value) {
if (value === undefined || value === "all" || value === "scrub")
return undefined;
if (Array.isArray(value))
return variableNameListProblem("inheritEnv", value);
return `inheritEnv must be "scrub", "all" or a list of variable names (got ${typeof value === "string" ? JSON.stringify(value) : value === null ? "null" : typeof value})`;
}
export function inheritedHostEnv(policy, findings) {
const problem = inheritEnvPolicyProblem(policy);
if (problem !== undefined)
throw new Error(problem);
return withholdOwnCredentials(policyArm(policy, findings), findings);
}
function policyArm(policy, findings) {
if (policy === "all")
return { ...process.env };
if (!Array.isArray(policy))
return scrubSecretEnv(process.env, findings);
const arm = {};
for (const k of policy)
if (process.env[k] !== undefined)
arm[k] = process.env[k];
return arm;
}
export function inheritEnvPolicyProblem(value) {
const shape = policyShapeProblem(value);
const { names } = withheldEnvOf();
if (shape !== undefined || !Array.isArray(value) || names.length === 0)
return shape;
const own = new Set(names.map(envNameKey));
const named = value.findIndex((k) => own.has(envNameKey(k)));
return named === -1 ? undefined : `inheritEnv[${named}] names ${value[named]}, which is this deployment's own credential and cannot be inherited by a child (a configuration error: no retry succeeds until the policy or the declaration changes)`;
}
let secretEnvScrubNoticeAnnounced = false;
export function __resetSecretEnvScrubNoticeAnnouncement() {
secretEnvScrubNoticeAnnounced = false;
}
export function announceSecretEnvScrub(onNotice, findings) {
if (findings.length === 0 || secretEnvScrubNoticeAnnounced)
return;
secretEnvScrubNoticeAnnounced = true;
const names = findings.map((f) => f.key);
const ownCredentials = findings.filter((f) => f.kind === "own-credential").map((f) => f.key);
const own = ownCredentials.length > 0 ? `; ${ownCredentials.length} of them this deployment's own credential(s) (${ownCredentials.join(", ")})` : "";
deliverEngineNotice(onNotice, {
code: "config.secret_env_scrubbed",
message: `secret_env_scrubbed: ${findings.length} env key(s) withheld from a spawned child's environment (${names.join(", ")})${own} — once per process; wire onNotice for the structured form`,
detail: { names, ...(ownCredentials.length > 0 ? { ownCredentials } : {}) },
});
}