@sema-agent/core
Version:
Stateless, task-oriented AI agent core
183 lines (182 loc) • 14.4 kB
JavaScript
export const URI_SCHEME_SOURCE = String.raw `[A-Za-z][A-Za-z0-9+.\-_]{0,63}`;
export const CREDENTIAL_MIN_CHARS = 8;
const VENDOR_BODY = `[A-Za-z0-9_-]{${CREDENTIAL_MIN_CHARS},}`;
const VALUE_END = String.raw `\s"';|&`;
const VALUE_CHAR = `[^${VALUE_END}]`;
const ESCAPE_DEPTH = 63;
const QUOTED_VALUE_MAX = 1024;
const PLAIN_QUOTES = [`"`, `'`, `"`, `'`, `"`, `'`, `%22`, `%27`];
const ESCAPED_QUOTES = [`"`, `'`, `x22`, `x27`, `u0022`, `u0027`, `042`, `047`];
const BACKSLASHES = String.raw `\\{1,${ESCAPE_DEPTH}}`;
const QUOTE = `(?:${PLAIN_QUOTES.join("|")}|${BACKSLASHES}(?:${ESCAPED_QUOTES.join("|")}))`;
const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE}))`;
const AFTER_CLOSING_QUOTE = String.raw `(?:[\s,}\]:]|["']|$)`;
const VALUE_RUN_END = `(?:(?<!\\\\)|(?!\\\\{0,${ESCAPE_DEPTH}}["']))`;
const GLUED_RUN = `${VALUE_CHAR}*${VALUE_RUN_END}`;
const pairedValue = (open, close) => `${open}(?!${AFTER_CLOSING_QUOTE})[^\\n]{0,${QUOTED_VALUE_MAX}}?${close}`;
const QUOTED_VALUE = [
...PLAIN_QUOTES.map((q) => pairedValue(q, q)),
...ESCAPED_QUOTES.map((q, i) => pairedValue(`(${BACKSLASHES})${q}`, `(?<!\\\\)(?:\\${i + 2})${q}`)),
`${QUOTE}(?!${AFTER_CLOSING_QUOTE})${GLUED_RUN}`,
].join("|");
const BEARER_TAIL = `(?:(?!${QUOTE})\\S${GLUED_RUN}|(?<==)(?:${QUOTED_VALUE}))`;
const HEADER_PARAM = String.raw `=[ \t]*(\\{0,${ESCAPE_DEPTH}})(["'])(?:(?!(?<!\\)\2\3)[^\r\n])*(?:(?<!\\)\2\3(?=[\s,;]|\\{0,${ESCAPE_DEPTH}}["']|$)|(?=[\r\n]|$))`;
const HEADER_VALUE = String.raw `(?:[A-Za-z][A-Za-z0-9!#$%&*+.^_|~-]*[ \t]+)?(?:${HEADER_PARAM}|[^\s"',]|[ \t]*,[ \t]*(?=[^\s"',])|[ \t]+(?==)|(?<==)[ \t]+(?=[^\s"',]))+${VALUE_RUN_END}`;
const headed = (body, flags = "g") => new RegExp(`${WORD_START}${body}`, flags);
export const CREDENTIAL_VOCABULARY = {
values: [
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/ },
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/ },
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/ },
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test)_[A-Za-z0-9]{24,}\b/ },
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`) },
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("glpat-[A-Za-z0-9_-]{20,}") },
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}") },
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`) },
{ id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("AIza[A-Za-z0-9_-]{20,}"), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprs]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprs]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/ },
{ id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/ },
{ id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("hf_[A-Za-z0-9]{20,}"), strict: /\bhf_(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{20,}\b/ },
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/ },
{ id: "private-key", kind: "private-key-block", source: "arg-summary", confidence: "high", match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)/g, strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/ },
{ id: "bearer-token", kind: "bearer-token", source: "arg-summary", confidence: "high", match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`), strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/ },
{ id: "authorization-header", kind: "authorization-header", source: "arg-summary", confidence: "high", match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi") },
{ id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: /(AccountKey=)[A-Za-z0-9+/]{40,}={0,2}/g, strict: /AccountKey=[A-Za-z0-9+/]{40,}={0,2}/ },
{ id: "keyword-value", kind: "keyword-value", source: "arg-summary", confidence: "medium", match: headed(String.raw `((?:token|secret|key|password|passwd|credential)["']?\s*[=:]\s*["']?)${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "gi") },
{ id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${URI_SCHEME_SOURCE}:\/\/)((?:[^/\s"'@]*@)+)`, "gi") },
{ id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: /([?&#](?:sig|signature|sas|code)=)[^&\s"'#]+/gi },
],
words: [
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE", "KEY"].map((word) => ({ word, position: "leading" })),
{ word: "AUTH", position: "leading", recordKeys: ["AUTHORIZATION", "AUTHENTICATION", "OAUTH", "AUTHN", "AUTHZ"] },
...["COOKIE", "PAT", "DSN", "WEBHOOK", "CREDENTIAL", "CREDENTIALS", "CREDS", "APIKEY", "ACCESSKEY", "SECRETKEY", "ACCOUNTKEY", "PRIVATEKEY", "AUTHKEY", "SSHKEY", "SIGNINGKEY", "MASTERKEY", "DEPLOYKEY", "ENCRYPTIONKEY", "PGPASSWORD", "SSHPASS"].map((word) => ({ word, position: "leading" })),
...["KEY", "SECRET", "PASSWORD", "CREDENTIAL"].map((word) => ({ word, position: "plural" })),
{ word: "TOKEN", position: "counted-plural" },
...["PWD", "PASS", "JWT"].map((word) => ({ word, position: "separated" })),
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE"].map((word) => ({ word, position: "embedded" })),
],
countWords: ["MAX", "OUTPUT", "INPUT", "THINKING", "NUM", "TOTAL"],
connectionStringName: /CONN(ECT(ION)?)?_?STR(ING)?S?(?=$|[_0-9])/i,
exactEnvNames: [
"ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_ARTIFACTS_API_TOKEN", "CLAUDE_CODE_MEMORY_API_TOKEN",
"CLAUDE_CODE_SLACK_TAG_TOKEN", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_FOUNDRY_API_KEY", "ANTHROPIC_FOUNDRY_AUTH_TOKEN",
"ANTHROPIC_AWS_API_KEY", "ANTHROPIC_CUSTOM_HEADERS", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
"AWS_BEARER_TOKEN_BEDROCK", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_GHA_CREDS_PATH", "AZURE_CLIENT_SECRET",
"IDENTITY_HEADER", "MSI_SECRET", "AZURE_CLIENT_CERTIFICATE_PATH", "AZURE_CLIENT_CERTIFICATE_PASSWORD",
"AZURE_PASSWORD", "AZURE_FEDERATED_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", "CLOUDSDK_AUTH_ACCESS_TOKEN",
"GOOGLE_OAUTH_ACCESS_TOKEN", "CLAUDE_CODE_OAUTH_REFRESH_TOKEN", "HF_TOKEN", "HUGGING_FACE_HUB_TOKEN",
"HUGGINGFACEHUB_API_TOKEN", "NODE_AUTH_TOKEN", "NUGET_AUTH_TOKEN", "CARGO_REGISTRY_TOKEN", "TWINE_PASSWORD",
"TWINE_USERNAME", "PYPI_TOKEN", "PYPI_API_TOKEN", "UV_PUBLISH_TOKEN", "UV_PUBLISH_PASSWORD", "UV_PUBLISH_USERNAME",
"FLIT_PASSWORD", "FLIT_USERNAME", "HATCH_INDEX_AUTH", "HATCH_INDEX_USER", "GEM_HOST_API_KEY", "MATURIN_PYPI_TOKEN",
"MATURIN_PASSWORD", "MATURIN_USERNAME", "CONAN_LOGIN_USERNAME", "CONAN_PASSWORD", "ANACONDA_API_TOKEN",
"BINSTAR_API_TOKEN", "VAULT_TOKEN", "VAULT_AUTH_TOKEN", "VAULT_ROLE_ID", "VAULT_SECRET_ID", "CONSUL_HTTP_TOKEN",
"CONSUL_HTTP_AUTH", "NOMAD_TOKEN", "NOMAD_HTTP_AUTH", "CI_REGISTRY_USER", "CI_DEPLOY_USER", "JF_USER",
"FASTLANE_SESSION", "MATCH_GIT_BASIC_AUTHORIZATION", "SONAR_TOKEN", "SONARQUBE_SCANNER_PARAMS",
"SONAR_SCANNER_JSON_PARAMS", "SLACK_WEBHOOK_URL", "SLACK_WEBHOOK", "DISCORD_WEBHOOK", "DISCORD_WEBHOOK_URL",
"TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URI", "ANTHROPIC_IDENTITY_TOKEN", "ANTHROPIC_IDENTITY_TOKEN_FILE",
"CLOUDSDK_AUTH_ACCESS_TOKEN_FILE", "CLOUDSDK_AUTH_AUTHORIZATION_TOKEN_FILE", "AZURE_AUTH_LOCATION",
"ACTIONS_ID_TOKEN_REQUEST_TOKEN", "ACTIONS_ID_TOKEN_REQUEST_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RUNTIME_URL",
"ALL_INPUTS", "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "ARTIFACTS_CREDENTIALPROVIDER_EXTERNAL_FEED_ENDPOINTS",
"VSS_NUGET_ACCESSTOKEN", "ARTIFACTS_CREDENTIALPROVIDER_ACCESSTOKEN", "COMPOSER_AUTH", "OVERRIDE_GITHUB_TOKEN",
"DEFAULT_WORKFLOW_TOKEN", "SSH_SIGNING_KEY",
],
nonSecretExactNames: [
"PARTITION_KEY", "SORT_KEY", "RANGE_KEY", "IDEMPOTENCY_KEY", "CACHE_KEY",
"DB_FOREIGN_KEYS", "JSON_SORT_KEYS",
"MAX_NEW_TOKENS", "MAX_COMPLETION_TOKENS", "AIDER_MAP_TOKENS", "AIDER_MAX_CHAT_HISTORY_TOKENS",
"MAX_BATCH_PREFILL_TOKENS", "MAX_WAITING_TOKENS", "MAX_TOP_N_TOKENS", "RATE_LIMIT_TOKENS",
"PROMPT_TOKENS", "CONTEXT_TOKENS",
],
nonSecretNameShape: /^GIT_CONFIG_KEY_[0-9][A-Za-z0-9_]*$/,
nonCredentialPrefix: /^AUTH0_/i,
tokenPrefix: /^(?:gh[opusr]_|github_pat_|glpat-|xox[abpr]-|sk-|pk-|AKIA|eyJ|ya29\.|npm_)/,
policy: { urlUserinfo: "keep-username" },
};
export const refuseLabel = (id) => id.replaceAll("-", "_");
export function refuseWriteView(vocabulary) {
return vocabulary.values.flatMap((row) => (row.strict === undefined ? [] : [{ label: refuseLabel(row.id), re: row.strict }]));
}
export function egressPassGroups(vocabulary) {
const groups = [];
for (const row of vocabulary.values) {
const last = groups[groups.length - 1];
if (row.pass !== undefined && last !== undefined && last[0].pass === row.pass)
last.push(row);
else
groups.push([row]);
}
return groups;
}
function credentialWordPattern(countWords, rows) {
const at = (position) => rows.filter((r) => r.position === position).map((r) => r.word);
const countedPlural = `(?<!(?:^|_)(?:${countWords.join("|")})_)(?:${at("counted-plural").join("|")})S`;
const [separated, embedded] = [at("separated"), at("embedded")];
const positions = [
`(^|_)(${at("leading").join("|")}|(${at("plural").join("|")})S|${countedPlural})`,
...(separated.length > 0 ? [`_(${separated.join("|")})`] : []),
...(embedded.length > 0 ? [`(${embedded.join("|")})`] : []),
];
const segmentWord = `(?:${[...at("leading"), ...at("plural").map((w) => `${w}S`), ...separated, countedPlural].join("|")})`;
return { credentialWord: new RegExp(`(${positions.join("|")})(?=$|[_0-9])`, "i"), segmentWord };
}
export function envNameView(vocabulary) {
const { credentialWord, segmentWord } = credentialWordPattern(vocabulary.countWords, vocabulary.words);
return {
exactNames: new Set(vocabulary.exactEnvNames.flatMap((name) => [name, `INPUT_${name}`])),
nonSecretExactNames: new Set(vocabulary.nonSecretExactNames),
nonSecretNameShape: vocabulary.nonSecretNameShape,
nonCredentialPrefix: vocabulary.nonCredentialPrefix,
credentialWord,
connectionStringName: vocabulary.connectionStringName,
segmentWord,
tokenPrefix: vocabulary.tokenPrefix,
};
}
export function splitCamelCase(name) {
return ["OAuth", "NextAuth"]
.reduce((text, word) => text.replaceAll(word, word[0] + word.slice(1).toLowerCase()), name)
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
.replace(/([A-Z])([A-Z][a-z])/g, "$1_$2");
}
export function recordKeyView(vocabulary) {
const rows = vocabulary.words.flatMap((w) => [w.word, ...(w.recordKeys ?? [])].map((word) => ({ word, position: w.position === "separated" ? "leading" : w.position })));
return {
credentialWord: credentialWordPattern(vocabulary.countWords, rows).credentialWord,
nonSecretExactNames: new Set(vocabulary.nonSecretExactNames),
nonSecretNameShape: vocabulary.nonSecretNameShape,
nonCredentialPrefix: vocabulary.nonCredentialPrefix,
};
}
const RECORD_KEYS = recordKeyView(CREDENTIAL_VOCABULARY);
function holdsCredentialWord(text, view) {
const words = text.replace(view.nonCredentialPrefix, "").replace(/-/g, "_");
return view.credentialWord.test(words) || view.credentialWord.test(splitCamelCase(words));
}
function isNonSecretRecordName(key, source, view) {
const spelled = key.replace(/-/g, "_");
const folded = spelled.toUpperCase();
if (view.nonSecretExactNames.has(folded))
return true;
if (source !== "header")
return false;
if (folded.length !== spelled.length)
return false;
for (const name of view.nonSecretExactNames) {
if (folded.endsWith(`_${name}`) && !holdsCredentialWord(spelled.slice(0, spelled.length - name.length - 1), view))
return true;
}
return false;
}
function readsAsCredential(key, source, view) {
if (view.nonSecretNameShape.test(key))
return false;
return holdsCredentialWord(key, view) && !isNonSecretRecordName(key, source, view);
}
export function isSensitiveRecordKey(key, view = RECORD_KEYS) {
return readsAsCredential(key, "member", view);
}
export function isSensitiveHeaderName(name, view = RECORD_KEYS) {
return readsAsCredential(name, "header", view);
}