genaiscript
Version:
A CLI for GenAIScript, a generative AI scripting framework.
1,962 lines (1,786 loc) • 177 kB
TypeScript
/**
* GenAIScript Ambient Type Definition File
* @version 1.142.15
*/
type OptionsOrString<TOptions extends string> = (string & {}) | TOptions
type ElementOrArray<T> = T | T[]
interface PromptGenerationConsole {
log(...data: any[]): void
warn(...data: any[]): void
debug(...data: any[]): void
error(...data: any[]): void
}
type DiagnosticSeverity = "error" | "warning" | "info"
interface Diagnostic {
filename: string
range: CharRange
severity: DiagnosticSeverity
message: string
/**
* suggested fix
*/
suggestion?: string
/**
* error or warning code
*/
code?: string
}
type Awaitable<T> = T | PromiseLike<T>
interface SerializedError {
name?: string
message?: string
stack?: string
cause?: unknown
code?: string
line?: number
column?: number
}
/**
* A color and icon to associate with the script.
* @see https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#branding
*/
interface PromptBranding {
/**
* Marketplace and web site branding
*/
branding?: {
/**
* The background color of the badge.
*/
color?:
| "white"
| "black"
| "yellow"
| "blue"
| "green"
| "orange"
| "red"
| "purple"
| "gray-dark"
/**
* Name of the Feather icon to use.
* @see https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#brandingicon
*/
icon?:
| "activity"
| "airplay"
| "alert-circle"
| "alert-octagon"
| "alert-triangle"
| "align-center"
| "align-justify"
| "align-left"
| "align-right"
| "anchor"
| "aperture"
| "archive"
| "arrow-down-circle"
| "arrow-down-left"
| "arrow-down-right"
| "arrow-down"
| "arrow-left-circle"
| "arrow-left"
| "arrow-right-circle"
| "arrow-right"
| "arrow-up-circle"
| "arrow-up-left"
| "arrow-up-right"
| "arrow-up"
| "at-sign"
| "award"
| "bar-chart-2"
| "bar-chart"
| "battery-charging"
| "battery"
| "bell-off"
| "bell"
| "bluetooth"
| "bold"
| "book-open"
| "book"
| "bookmark"
| "box"
| "briefcase"
| "calendar"
| "camera-off"
| "camera"
| "cast"
| "check-circle"
| "check-square"
| "check"
| "chevron-down"
| "chevron-left"
| "chevron-right"
| "chevron-up"
| "chevrons-down"
| "chevrons-left"
| "chevrons-right"
| "chevrons-up"
| "circle"
| "clipboard"
| "clock"
| "cloud-drizzle"
| "cloud-lightning"
| "cloud-off"
| "cloud-rain"
| "cloud-snow"
| "cloud"
| "code"
| "command"
| "compass"
| "copy"
| "corner-down-left"
| "corner-down-right"
| "corner-left-down"
| "corner-left-up"
| "corner-right-down"
| "corner-right-up"
| "corner-up-left"
| "corner-up-right"
| "cpu"
| "credit-card"
| "crop"
| "crosshair"
| "database"
| "delete"
| "disc"
| "dollar-sign"
| "download-cloud"
| "download"
| "droplet"
| "edit-2"
| "edit-3"
| "edit"
| "external-link"
| "eye-off"
| "eye"
| "fast-forward"
| "feather"
| "file-minus"
| "file-plus"
| "file-text"
| "file"
| "film"
| "filter"
| "flag"
| "folder-minus"
| "folder-plus"
| "folder"
| "gift"
| "git-branch"
| "git-commit"
| "git-merge"
| "git-pull-request"
| "globe"
| "grid"
| "hard-drive"
| "hash"
| "headphones"
| "heart"
| "help-circle"
| "home"
| "image"
| "inbox"
| "info"
| "italic"
| "layers"
| "layout"
| "life-buoy"
| "link-2"
| "link"
| "list"
| "loader"
| "lock"
| "log-in"
| "log-out"
| "mail"
| "map-pin"
| "map"
| "maximize-2"
| "maximize"
| "menu"
| "message-circle"
| "message-square"
| "mic-off"
| "mic"
| "minimize-2"
| "minimize"
| "minus-circle"
| "minus-square"
| "minus"
| "monitor"
| "moon"
| "more-horizontal"
| "more-vertical"
| "move"
| "music"
| "navigation-2"
| "navigation"
| "octagon"
| "package"
| "paperclip"
| "pause-circle"
| "pause"
| "percent"
| "phone-call"
| "phone-forwarded"
| "phone-incoming"
| "phone-missed"
| "phone-off"
| "phone-outgoing"
| "phone"
| "pie-chart"
| "play-circle"
| "play"
| "plus-circle"
| "plus-square"
| "plus"
| "pocket"
| "power"
| "printer"
| "radio"
| "refresh-ccw"
| "refresh-cw"
| "repeat"
| "rewind"
| "rotate-ccw"
| "rotate-cw"
| "rss"
| "save"
| "scissors"
| "search"
| "send"
| "server"
| "settings"
| "share-2"
| "share"
| "shield-off"
| "shield"
| "shopping-bag"
| "shopping-cart"
| "shuffle"
| "sidebar"
| "skip-back"
| "skip-forward"
| "slash"
| "sliders"
| "smartphone"
| "speaker"
| "square"
| "star"
| "stop-circle"
| "sun"
| "sunrise"
| "sunset"
| "table"
| "tablet"
| "tag"
| "target"
| "terminal"
| "thermometer"
| "thumbs-down"
| "thumbs-up"
| "toggle-left"
| "toggle-right"
| "trash-2"
| "trash"
| "trending-down"
| "trending-up"
| "triangle"
| "truck"
| "tv"
| "type"
| "umbrella"
| "underline"
| "unlock"
| "upload-cloud"
| "upload"
| "user-check"
| "user-minus"
| "user-plus"
| "user-x"
| "user"
| "users"
| "video-off"
| "video"
| "voicemail"
| "volume-1"
| "volume-2"
| "volume-x"
| "volume"
| "watch"
| "wifi-off"
| "wifi"
| "wind"
| "x-circle"
| "x-square"
| "x"
| "zap-off"
| "zap"
| "zoom-in"
| "zoom-out"
}
}
interface PromptDefinition {
/**
* Based on file name.
*/
id: string
/**
* Something like "Summarize children", show in UI.
*/
title?: string
/**
* Longer description of the prompt. Shows in UI grayed-out.
*/
description?: string
/**
* Groups template in UI
*/
group?: string
/**
* List of tools defined in the script
*/
defTools?: { id: string; description: string; kind: "tool" | "agent" }[]
}
interface PromptLike extends PromptDefinition {
/**
* File where the prompt comes from (if any).
*/
filename?: string
/**
* The actual text of the prompt template.
* Only used for system prompts.
*/
text?: string
/**
* The text of the prompt JS source code.
*/
jsSource?: string
/**
* Resolved system ids
*/
resolvedSystem?: SystemPromptInstance[]
/**
* Inferred input schema for parameters
*/
inputSchema?: JSONSchemaObject
}
type SystemPromptId = OptionsOrString<
| "synthlang"
| "system"
| "system.agent_data"
| "system.agent_docs"
| "system.agent_fs"
| "system.agent_git"
| "system.agent_github"
| "system.agent_interpreter"
| "system.agent_mcp"
| "system.agent_planner"
| "system.agent_user_input"
| "system.agent_video"
| "system.agent_web"
| "system.agent_z3"
| "system.annotations"
| "system.assistant"
| "system.chain_of_draft"
| "system.changelog"
| "system.cooperation"
| "system.diagrams"
| "system.diff"
| "system.do_not_explain"
| "system.english"
| "system.explanations"
| "system.fetch"
| "system.files"
| "system.files_schema"
| "system.fs_ask_file"
| "system.fs_data_query"
| "system.fs_diff_files"
| "system.fs_find_files"
| "system.fs_read_file"
| "system.git"
| "system.git_diff"
| "system.git_info"
| "system.github_actions"
| "system.github_files"
| "system.github_info"
| "system.github_issues"
| "system.github_pulls"
| "system.math"
| "system.mcp"
| "system.md_find_files"
| "system.md_frontmatter"
| "system.meta_prompt"
| "system.meta_schema"
| "system.node_info"
| "system.node_test"
| "system.output_ini"
| "system.output_json"
| "system.output_markdown"
| "system.output_plaintext"
| "system.output_yaml"
| "system.planner"
| "system.python"
| "system.python_code_interpreter"
| "system.python_types"
| "system.retrieval_fuzz_search"
| "system.retrieval_vector_search"
| "system.retrieval_web_search"
| "system.safety_canary_word"
| "system.safety_harmful_content"
| "system.safety_jailbreak"
| "system.safety_protected_material"
| "system.safety_ungrounded_content_summarization"
| "system.safety_validate_harmful_content"
| "system.schema"
| "system.tasks"
| "system.technical"
| "system.think"
| "system.today"
| "system.tool_calls"
| "system.tools"
| "system.transcribe"
| "system.typescript"
| "system.user_input"
| "system.user-tool"
| "system.video"
| "system.vision_ask_images"
| "system.z3"
| "system.zero_shot_cot"
>
type SystemPromptInstance = {
id: SystemPromptId
parameters?: Record<string, string | boolean | number | object | any>
vars?: Record<string, string | boolean | number | object | any>
}
type SystemToolId = OptionsOrString<
| "agent_data"
| "agent_docs"
| "agent_fs"
| "agent_git"
| "agent_github"
| "agent_interpreter"
| "agent_planner"
| "agent_user_input"
| "agent_video"
| "agent_web"
| "agent_z3"
| "fetch"
| "fs_ask_file"
| "fs_data_query"
| "fs_diff_files"
| "fs_find_files"
| "fs_read_file"
| "git_branch_current"
| "git_branch_default"
| "git_branch_list"
| "git_diff"
| "git_last_tag"
| "git_list_commits"
| "git_status"
| "github_actions_job_logs_diff"
| "github_actions_job_logs_get"
| "github_actions_jobs_list"
| "github_actions_workflows_list"
| "github_files_get"
| "github_files_list"
| "github_issues_comments_list"
| "github_issues_get"
| "github_issues_list"
| "github_pulls_get"
| "github_pulls_list"
| "github_pulls_review_comments_list"
| "math_eval"
| "md_find_files"
| "md_read_frontmatter"
| "meta_prompt"
| "meta_schema"
| "my_random"
| "node_test"
| "python_code_interpreter_copy_files_to_container"
| "python_code_interpreter_read_file"
| "python_code_interpreter_run"
| "retrieval_fuzz_search"
| "retrieval_vector_search"
| "retrieval_web_search"
| "think"
| "transcribe"
| "user_input_confirm"
| "user_input_select"
| "user_input_text"
| "video_extract_audio"
| "video_extract_clip"
| "video_extract_frames"
| "video_probe"
| "vision_ask_images"
| "z3"
>
type FileMergeHandler = (
filename: string,
label: string,
before: string,
generated: string
) => Awaitable<string>
interface PromptOutputProcessorResult {
/**
* Updated text
*/
text?: string
/**
* Generated files from the output
*/
files?: Record<string, string>
/**
* User defined errors
*/
annotations?: Diagnostic[]
}
type PromptOutputProcessorHandler = (
output: GenerationOutput
) =>
| PromptOutputProcessorResult
| Promise<PromptOutputProcessorResult>
| undefined
| Promise<undefined>
| void
| Promise<void>
type PromptTemplateResponseType =
| "text"
| "json"
| "yaml"
| "markdown"
| "json_object"
| "json_schema"
| undefined
type ModelType = OptionsOrString<
| "large"
| "small"
| "tiny"
| "long"
| "vision"
| "vision_small"
| "reasoning"
| "reasoning_small"
| "openai:gpt-4.1"
| "openai:gpt-4.1-mini"
| "openai:gpt-4.1-nano"
| "openai:gpt-4o"
| "openai:gpt-4o-mini"
| "openai:gpt-3.5-turbo"
| "openai:o3-mini"
| "openai:o3-mini:low"
| "openai:o3-mini:medium"
| "openai:o3-mini:high"
| "openai:o1"
| "openai:o1-mini"
| "openai:o1-preview"
| "github:openai/gpt-4.1"
| "github:openai/gpt-4o"
| "github:openai/gpt-4o-mini"
| "github:openai/o1"
| "github:openai/o1-mini"
| "github:openai/o3-mini"
| "github:openai/o3-mini:low"
| "github:microsoft/mai-ds-r1"
| "github:deepseek/deepseek-v3"
| "github:deepseek/deepseek-r1"
| "github:microsoft/phi-4"
| "github_copilot_chat:current"
| "github_copilot_chat:gpt-3.5-turbo"
| "github_copilot_chat:gpt-4o-mini"
| "github_copilot_chat:gpt-4o-2024-11-20"
| "github_copilot_chat:gpt-4"
| "github_copilot_chat:o1"
| "github_copilot_chat:o1:low"
| "github_copilot_chat:o1:medium"
| "github_copilot_chat:o1:high"
| "github_copilot_chat:o3-mini"
| "github_copilot_chat:o3-mini:low"
| "github_copilot_chat:o3-mini:medium"
| "github_copilot_chat:o3-mini:high"
| "azure:gpt-4o"
| "azure:gpt-4o-mini"
| "azure:o1"
| "azure:o1-mini"
| "azure:o1-preview"
| "azure:o3-mini"
| "azure:o3-mini:low"
| "azure:o3-mini:medium"
| "azure:o3-mini:high"
| "azure_ai_inference:gpt-4.1"
| "azure_ai_inference:gpt-4o"
| "azure_ai_inference:gpt-4o-mini"
| "azure_ai_inference:o1"
| "azure_ai_inference:o1-mini"
| "azure_ai_inference:o1-preview"
| "azure_ai_inference:o3-mini"
| "azure_ai_inference:o3-mini:low"
| "azure_ai_inference:o3-mini:medium"
| "azure_ai_inference:o3-mini:high"
| "azure_ai_inference:deepSeek-v3"
| "azure_ai_inference:deepseek-r1"
| "ollama:gemma3:4b"
| "ollama:marco-o1"
| "ollama:tulu3"
| "ollama:athene-v2"
| "ollama:opencoder"
| "ollama:qwen2.5-coder"
| "ollama:llama3.2-vision"
| "ollama:llama3.2"
| "ollama:phi4"
| "ollama:phi3.5"
| "ollama:deepseek-r1:1.5b"
| "ollama:deepseek-r1:7b"
| "ollama:olmo2:7b"
| "ollama:command-r7b:7b"
| "anthropic:claude-3-7-sonnet-latest"
| "anthropic:claude-3-7-sonnet-latest:low"
| "anthropic:claude-3-7-sonnet-latest:medium"
| "anthropic:claude-3-7-sonnet-latest:high"
| "anthropic:claude-3-7-sonnet-20250219"
| "anthropic:claude-3-5-sonnet-latest"
| "anthropic:claude-3-5-sonnet-20240620"
| "anthropic:claude-3-opus-20240229"
| "anthropic:claude-3-sonnet-20240229"
| "anthropic:claude-3-haiku-20240307"
| "anthropic:claude-2.1"
| "anthropic_bedrock:anthropic.claude-3-7-sonnet-20250219-v1:0"
| "anthropic_bedrock:anthropic.claude-3-7-sonnet-20250219-v1:0:low"
| "anthropic_bedrock:anthropic.claude-3-7-sonnet-20250219-v1:0:medium"
| "anthropic_bedrock:anthropic.claude-3-7-sonnet-20250219-v1:0:high"
| "anthropic_bedrock:anthropic.claude-3-5-haiku-20241022-v1:0"
| "anthropic_bedrock:anthropic.claude-3-5-sonnet-20241022-v2:0"
| "anthropic_bedrock:anthropic.claude-3-5-sonnet-20240620-v1:0"
| "anthropic_bedrock:anthropic.claude-3-opus-20240229-v1:0"
| "anthropic_bedrock:anthropic.claude-3-sonnet-20240229-v1:0"
| "anthropic_bedrock:anthropic.claude-3-haiku-20240307-v1:0"
| "huggingface:microsoft/Phi-3-mini-4k-instruct"
| "jan:llama3.2-3b-instruct"
| "google:gemini-2.0-flash-exp"
| "google:gemini-2.0-flash-thinking-exp-1219"
| "google:gemini-1.5-flash"
| "google:gemini-1.5-flash-latest"
| "google:gemini-1.5-flash-8b"
| "google:gemini-1.5-flash-8b-latest"
| "google:gemini-1.5-pro"
| "google:gemini-1.5-pro-latest"
| "mistral:mistral-large-latest"
| "mistral:mistral-small-latest"
| "mistral:pixtral-large-latest"
| "mistral:codestral-latest"
| "mistral:nemo"
| "alibaba:qwen-turbo"
| "alibaba:qwen-max"
| "alibaba:qwen-plus"
| "alibaba:qwen2-72b-instruct"
| "alibaba:qwen2-57b-a14b-instruct"
| "deepseek:deepseek-chat"
// | "transformers:onnx-community/Qwen2.5-0.5B-Instruct:q4"
// | "transformers:HuggingFaceTB/SmolLM2-1.7B-Instruct:q4f16"
| "llamafile"
| "sglang"
| "vllm"
| "echo"
| "none"
>
type EmbeddingsModelType = OptionsOrString<
| "openai:text-embedding-3-small"
| "openai:text-embedding-3-large"
| "openai:text-embedding-ada-002"
| "github:text-embedding-3-small"
| "github:text-embedding-3-large"
| "azure:text-embedding-3-small"
| "azure:text-embedding-3-large"
| "azure_ai_inference:text-embedding-3-small"
| "azure_ai_inference:text-embedding-3-large"
| "ollama:nomic-embed-text"
| "google:text-embedding-004"
| "huggingface:nomic-ai/nomic-embed-text-v1.5"
>
type ModelSmallType = OptionsOrString<
| "openai:gpt-4o-mini"
| "github:openai/gpt-4o-mini"
| "azure:gpt-4o-mini"
| "github:microsoft/phi-4"
>
type ModelVisionType = OptionsOrString<
| "openai:gpt-4o"
| "github:openai/gpt-4o"
| "azure:gpt-4o"
| "azure:gpt-4o-mini"
>
type ModelImageGenerationType = OptionsOrString<
"openai:gpt-image-1" | "openai:dall-e-2" | "openai:dall-e-3"
>
type ModelProviderType = OptionsOrString<
| "openai"
| "azure"
| "azure_serverless"
| "azure_serverless_models"
| "anthropic"
| "anthropic_bedrock"
| "google"
| "huggingface"
| "mistral"
| "alibaba"
| "github"
| "transformers"
| "ollama"
| "lmstudio"
| "jan"
| "sglang"
| "vllm"
| "llamafile"
| "litellm"
| "github_copilot_chat"
| "deepseek"
| "whisperasr"
| "echo"
>
interface ModelConnectionOptions {
/**
* Which LLM model by default or for the `large` alias.
*/
model?: ModelType
}
interface ModelAliasesOptions extends ModelConnectionOptions {
/**
* Configure the `small` model alias.
*/
smallModel?: ModelSmallType
/**
* Configure the `vision` model alias.
*/
visionModel?: ModelVisionType
/**
* A list of model aliases to use.
*/
modelAliases?: Record<string, string>
}
type ReasoningEffortType = "high" | "medium" | "low"
type ChatToolChoice =
| "none"
| "auto"
| "required"
| {
/**
* The name of the function to call.
*/
name: string
}
interface ModelOptions
extends ModelConnectionOptions,
ModelTemplateOptions,
CacheOptions {
/**
* Temperature to use. Higher temperature means more hallucination/creativity.
* Range 0.0-2.0.
*
* @default 0.2
*/
temperature?: number
/**
* Enables fallback tools mode
*/
fallbackTools?: boolean
/**
* OpenAI o* reasoning models support a reasoning effort parameter.
* For Clause, these are mapped to thinking budget tokens
*/
reasoningEffort?: ReasoningEffortType
/**
* A list of keywords that should be found in the output.
*/
choices?: ElementOrArray<
string | { token: string | number; weight?: number }
>
/**
* Returns the log probabilities of the each tokens. Not supported in all models.
*/
logprobs?: boolean
/**
* Number of alternate token logprobs to generate, up to 5. Enables logprobs.
*/
topLogprobs?: number
/**
* Specifies the type of output. Default is plain text.
* - `text` enables plain text mode (through system prompts)
* - `json` enables JSON mode (through system prompts)
* - `yaml` enables YAML mode (through system prompts)
* - `json_object` enables JSON mode (native)
* - `json_schema` enables structured outputs (native)
* Use `responseSchema` to specify an output schema.
*/
responseType?: PromptTemplateResponseType
/**
* JSON object schema for the output. Enables the `json_object` output mode by default.
*/
responseSchema?: PromptParametersSchema | JSONSchema
/**
* “Top_p” or nucleus sampling is a setting that decides how many possible words to consider.
* A high “top_p” value means the model looks at more possible words, even the less likely ones,
* which makes the generated text more diverse.
*/
topP?: number
/**
* Maximum number of completion tokens
*
*/
maxTokens?: number
/**
* Tool selection strategy. Default is 'auto'.
*/
toolChoice?: ChatToolChoice
/**
* Maximum number of tool calls to make.
*/
maxToolCalls?: number
/**
* Maximum number of data repairs to attempt.
*/
maxDataRepairs?: number
/**
* A deterministic integer seed to use for the model.
*/
seed?: number
/**
* A list of model ids and their maximum number of concurrent requests.
*/
modelConcurrency?: Record<string, number>
}
interface EmbeddingsModelOptions {
/**
* LLM model to use for embeddings.
*/
embeddingsModel?: EmbeddingsModelType
}
interface PromptSystemOptions extends PromptSystemSafetyOptions {
/**
* List of system script ids used by the prompt.
*/
system?: ElementOrArray<SystemPromptId | SystemPromptInstance>
/**
* List of tools used by the prompt.
*/
tools?: ElementOrArray<SystemToolId>
/**
* List of system to exclude from the prompt.
*/
excludedSystem?: ElementOrArray<SystemPromptId>
/**
* MCP server configuration. The tools will be injected into the prompt.
*/
mcpServers?: McpServersConfig
/**
* MCP agent configuration. Each mcp server will be wrapped with an agent.
*/
mcpAgentServers?: McpAgentServersConfig
}
interface ScriptRuntimeOptions extends LineNumberingOptions {
/**
* Secrets required by the prompt
*/
secrets?: string[]
}
type PromptJSONParameterType<T> = T & { required?: boolean }
type PromptParameterType =
| string
| number
| boolean
| object
| PromptJSONParameterType<JSONSchemaNumber>
| PromptJSONParameterType<JSONSchemaString>
| PromptJSONParameterType<JSONSchemaBoolean>
type PromptParametersSchema = Record<
string,
PromptParameterType | [PromptParameterType]
>
type PromptParameters = Record<string, string | number | boolean | object>
type PromptAssertion = {
// How heavily to weigh the assertion. Defaults to 1.0
weight?: number
/**
* The transformation to apply to the output before checking the assertion.
*/
transform?: string
} & (
| {
// type of assertion
type:
| "icontains"
| "not-icontains"
| "equals"
| "not-equals"
| "starts-with"
| "not-starts-with"
// The expected value
value: string
}
| {
// type of assertion
type:
| "contains-all"
| "not-contains-all"
| "contains-any"
| "not-contains-any"
| "icontains-all"
| "not-icontains-all"
// The expected values
value: string[]
}
| {
// type of assertion
type: "levenshtein" | "not-levenshtein"
// The expected value
value: string
// The threshold value
threshold?: number
}
| {
type: "javascript"
/**
* JavaScript expression to evaluate.
*/
value: string
/**
* Optional threshold if the javascript expression returns a number
*/
threshold?: number
}
)
interface PromptTest {
/**
* Short name of the test
*/
name?: string
/**
* Description of the test.
*/
description?: string
/**
* List of files to apply the test to.
*/
files?: ElementOrArray<string>
/**
* List of in-memory files to apply the test to.
*/
workspaceFiles?: ElementOrArray<WorkspaceFile>
/**
* Extra set of variables for this scenario
*/
vars?: Record<string, string | boolean | number>
/**
* LLM output matches a given rubric, using a Language Model to grade output.
*/
rubrics?: ElementOrArray<string>
/**
* LLM output adheres to the given facts, using Factuality method from OpenAI evaluation.
*/
facts?: ElementOrArray<string>
/**
* List of keywords that should be contained in the LLM output.
*/
keywords?: ElementOrArray<string>
/**
* List of keywords that should not be contained in the LLM output.
*/
forbidden?: ElementOrArray<string>
/**
* Additional deterministic assertions.
*/
asserts?: ElementOrArray<PromptAssertion>
/**
* Determines what kind of output is sent back to the test engine. Default is "text".
*/
format?: "text" | "json"
}
/**
* Configure promptfoo redteam plugins
*/
interface PromptRedteam {
/**
* The `purpose` property is used to guide the attack generation process. It should be as clear and specific as possible.
* Include the following information:
* - Who the user is and their relationship to the company
* - What data the user has access to
* - What data the user does not have access to
* - What actions the user can perform
* - What actions the user cannot perform
* - What systems the agent has access to
* @link https://www.promptfoo.dev/docs/red-team/troubleshooting/attack-generation/
*/
purpose: string
/**
* Redteam identifier used for reporting purposes
*/
label?: string
/**
* Default number of inputs to generate for each plugin.
* The total number of tests will be `(numTests * plugins.length * (1 + strategies.length) * languages.length)`
* Languages.length is 1 by default, but is added when the multilingual strategy is used.
*/
numTests?: number
/**
* List of languages to target. Default is English.
*/
language?: string
/**
* Red team plugin list
* @link https://www.promptfoo.dev/docs/red-team/owasp-llm-top-10/
*/
plugins?: ElementOrArray<
OptionsOrString<
| "default"
| "nist:ai:measure"
| "owasp:llm"
| "owasp:api"
| "mitre:atlas"
| "owasp:llm:01"
| "owasp:llm:02"
| "owasp:llm:04"
| "owasp:llm:06"
| "owasp:llm:09"
| "contracts"
| "divergent-repetition"
| "excessive-agency"
| "hallucination"
| "harmful:chemical-biological-weapons"
| "harmful:child-exploitation"
| "harmful:copyright-violations"
| "harmful:cybercrime"
| "harmful:cybercrime:malicious-code"
| "harmful:graphic-content"
| "harmful:harassment-bullying"
| "harmful:hate"
| "harmful:illegal-activities"
| "harmful:illegal-drugs"
| "harmful:illegal-drugs:meth"
| "harmful:indiscriminate-weapons"
| "harmful:insults"
| "harmful:intellectual-property"
| "harmful:misinformation-disinformation"
| "harmful:non-violent-crime"
| "harmful:privacy"
| "harmful:profanity"
| "harmful:radicalization"
| "harmful:self-harm"
| "harmful:sex-crime"
| "harmful:sexual-content"
| "harmful:specialized-advice"
| "harmful:unsafe-practices"
| "harmful:violent-crime"
| "harmful:weapons:ied"
| "hijacking"
| "pii:api-db"
| "pii:direct"
| "pii:session"
| "pii:social"
| "politics"
>
>
/**
* Adversary prompt generation strategies
*/
strategies?: ElementOrArray<
OptionsOrString<
| "default"
| "basic"
| "jailbreak"
| "jailbreak:composite"
| "base64"
| "jailbreak"
| "prompt-injection"
>
>
}
/**
* Different ways to render a fence block.
*/
type FenceFormat = "markdown" | "xml" | "none"
interface FenceFormatOptions {
/**
* Formatting of code sections
*/
fenceFormat?: FenceFormat
}
interface ModelTemplateOptions extends FenceFormatOptions {
/**
* Budget of tokens to apply the prompt flex renderer.
*/
flexTokens?: number
}
interface McpToolAnnotations {
/**
* Annotations for MCP tools
* @link https://modelcontextprotocol.io/docs/concepts/tools#available-tool-annotations
*/
annotations?: {
/**
* If true, indicates the tool does not modify its environment
*/
readOnlyHint?: boolean
/**
* If true, the tool may perform destructive updates (only meaningful when readOnlyHint is false)
*/
destructiveHint?: boolean
/**
* If true, calling the tool repeatedly with the same arguments has no additional effect (only meaningful when readOnlyHint is false)
*/
idempotentHint?: boolean
/**
* If true, the tool may interact with an “open world” of external entities
*/
openWorldHint?: boolean
}
}
interface MetadataOptions {
/**
* Set of 16 key-value pairs that can be attached to an object.
* This can be useful for storing additional information about the object in a structured format, and querying for objects via API or the dashboard.
* Keys are strings with a maximum length of 64 characters. Values are strings with a maximum length of 512 characters.
*/
metadata?: Record<string, string>
}
interface PromptScript
extends PromptLike,
PromptBranding,
ModelOptions,
ModelAliasesOptions,
PromptSystemOptions,
EmbeddingsModelOptions,
ContentSafetyOptions,
SecretDetectionOptions,
GitIgnoreFilterOptions,
ScriptRuntimeOptions,
McpToolAnnotations,
MetadataOptions {
/**
* Which provider to prefer when picking a model.
*/
provider?: ModelProviderType
/**
* Additional template parameters that will populate `env.vars`
*/
parameters?: PromptParametersSchema
/**
* A file path or list of file paths or globs.
* The content of these files will be by the files selected in the UI by the user or the cli arguments.
*/
files?: ElementOrArray<string>
/**
* A comma separated list of file extensions to accept.
*/
accept?: OptionsOrString<".md,.mdx" | "none">
/**
* Extra variable values that can be used to configure system prompts.
*/
vars?: Record<string, string>
/**
* Tests to validate this script.
*/
tests?: ElementOrArray<string | PromptTest>
/**
* Models to use with tests
*/
testModels?: ElementOrArray<ModelType | ModelAliasesOptions>
/**
* LLM vulnerability checks
*/
redteam?: PromptRedteam
/**
* Don't show it to the user in lists. Template `system.*` are automatically unlisted.
*/
unlisted?: boolean
/**
* Set if this is a system prompt.
*/
isSystem?: boolean
}
/**
* Represent a workspace file and optional content.
*/
interface WorkspaceFile {
/**
* Name of the file, relative to project root.
*/
filename: string
/**
* Content mime-type if known
*/
type?: string
/**
* Encoding of the content
*/
encoding?: "base64"
/**
* Content of the file.
*/
content?: string
/**
* Size in bytes if known
*/
size?: number
}
interface WorkspaceFileWithScore extends WorkspaceFile {
/**
* Score allocated by search algorithm
*/
score?: number
}
interface ToolDefinition {
/**
* The name of the function to be called. Must be a-z, A-Z, 0-9, or contain
* underscores and dashes, with a maximum length of 64.
*/
name: string
/**
* A description of what the function does, used by the model to choose when and
* how to call the function.
*/
description?: string
/**
* The parameters the functions accepts, described as a JSON Schema object. See the
* [guide](https://platform.openai.com/docs/guides/text-generation/function-calling)
* for examples, and the
* [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for
* documentation about the format.
*
* Omitting `parameters` defines a function with an empty parameter list.
*/
parameters?: JSONSchema
}
/**
* Interface representing an output trace with various logging and tracing methods.
* Extends the `ToolCallTrace` interface.
*/
interface OutputTrace extends ToolCallTrace {
/**
* Logs a heading message at the specified level.
* @param level - The level of the heading.
* @param message - The heading message.
*/
heading(level: number, message: string): void
/**
* Logs an image with an optional caption.
* @param url - The URL of the image.
* @param caption - The optional caption for the image.
*/
image(url: BufferLike, caption?: string): Promise<void>
/**
* Logs a markdown table
* @param rows
*/
table(rows: object[]): void
/**
* Computes and renders diff between two files.
*/
diff(
left: string | WorkspaceFile,
right: string | WorkspaceFile,
options?: { context?: number }
): void
/**
* Logs a result item with a boolean value and a message.
* @param value - The boolean value of the result item.
* @param message - The message for the result item.
*/
resultItem(value: boolean, message: string): void
/**
* Starts a trace with details in markdown format.
* @param title - The title of the trace.
* @param options - Optional settings for the trace.
* @returns A `MarkdownTrace` instance.
*/
startTraceDetails(
title: string,
options?: { expanded?: boolean }
): OutputTrace
/**
* Appends content to the trace.
* @param value - The content to append.
*/
appendContent(value: string): void
/**
* Starts a details section in the trace.
* @param title - The title of the details section.
* @param options - Optional settings for the details section.
*/
startDetails(
title: string,
options?: { success?: boolean; expanded?: boolean }
): void
/**
* Ends the current details section in the trace.
*/
endDetails(): void
/**
* Logs a video with a name, file path, and optional alt text.
* @param name - The name of the video.
* @param filepath - The file path of the video.
* @param alt - The optional alt text for the video.
*/
video(name: string, filepath: string, alt?: string): void
/**
* Logs an audio file
* @param name
* @param filepath
* @param alt
*/
audio(name: string, filepath: string, alt?: string): void
/**
* Logs a details section with a title and body.
* @param title - The title of the details section.
* @param body - The body content of the details section, can be a string or an object.
* @param options - Optional settings for the details section.
*/
details(
title: string,
body: string | object,
options?: { success?: boolean; expanded?: boolean }
): void
/**
* Logs a fenced details section with a title, body, and optional content type.
* @param title - The title of the details section.
* @param body - The body content of the details section, can be a string or an object.
* @param contentType - The optional content type of the body.
* @param options - Optional settings for the details section.
*/
detailsFenced(
title: string,
body: string | object,
contentType?: string,
options?: { expanded?: boolean }
): void
/**
* Logs an item with a name, value, and optional unit.
* @param name - The name of the item.
* @param value - The value of the item.
* @param unit - The optional unit of the value.
*/
itemValue(name: string, value: any, unit?: string): void
/**
* Adds a url link item
* @param name name url
* @param url url. If missing, name is treated as the url.
*/
itemLink(name: string, url?: string | URL, title?: string): void
/**
* Writes a paragraph of text with empty lines before and after.
* @param text paragraph to write
*/
p(text: string): void
/**
* Logs a warning message.
* @param msg - The warning message to log.
*/
warn(msg: string): void
/**
* Logs a caution message.
* @param msg - The caution message to log.
*/
caution(msg: string): void
/**
* Logs a note message.
* @param msg - The note message to log.
*/
note(msg: string): void
/**
* Logs an error object
* @param err
*/
error(message: string, error?: unknown): void
}
/**
* Interface representing a tool call trace for logging various types of messages.
*/
interface ToolCallTrace {
/**
* Logs a general message.
* @param message - The message to log.
*/
log(message: string): void
/**
* Logs an item message.
* @param message - The item message to log.
*/
item(message: string): void
/**
* Logs a tip message.
* @param message - The tip message to log.
*/
tip(message: string): void
/**
* Logs a fenced message, optionally specifying the content type.
* @param message - The fenced message to log.
* @param contentType - The optional content type of the message.
*/
fence(message: string | unknown, contentType?: string): void
}
/**
* Position (line, character) in a file. Both are 0-based.
*/
type CharPosition = [number, number]
/**
* Describes a run of text.
*/
type CharRange = [CharPosition, CharPosition]
/**
* 0-based line numbers.
*/
type LineRange = [number, number]
interface FileEdit {
type: string
filename: string
label?: string
validated?: boolean
}
interface ReplaceEdit extends FileEdit {
type: "replace"
range: CharRange | LineRange
text: string
}
interface InsertEdit extends FileEdit {
type: "insert"
pos: CharPosition | number
text: string
}
interface DeleteEdit extends FileEdit {
type: "delete"
range: CharRange | LineRange
}
interface CreateFileEdit extends FileEdit {
type: "createfile"
overwrite?: boolean
ignoreIfExists?: boolean
text: string
}
type Edits = InsertEdit | ReplaceEdit | DeleteEdit | CreateFileEdit
interface ToolCallContent {
type?: "content"
content: string
edits?: Edits[]
}
type ToolCallOutput =
| string
| number
| boolean
| ToolCallContent
| ShellOutput
| WorkspaceFile
| RunPromptResult
| SerializedError
| undefined
interface WorkspaceFileCache<K, V> {
/**
* Name of the cache
*/
name: string
/**
* Gets the value associated with the key, or undefined if there is none.
* @param key
*/
get(key: K): Promise<V | undefined>
/**
* Sets the value associated with the key.
* @param key
* @param value
*/
set(key: K, value: V): Promise<void>
/**
* List the values in the cache.
*/
values(): Promise<V[]>
/**
* Gets the sha of the key
* @param key
*/
getSha(key: K): Promise<string>
/**
* Gets an existing value or updates it with the updater function.
*/
getOrUpdate(
key: K,
updater: () => Promise<V>,
validator?: (val: V) => boolean
): Promise<{ key: string; value: V; cached?: boolean }>
}
interface WorkspaceGrepOptions extends FilterGitFilesOptions {
/**
* List of paths to
*/
path?: ElementOrArray<string>
/**
* list of filename globs to search. !-prefixed globs are excluded. ** are not supported.
*/
glob?: ElementOrArray<string>
/**
* Read file content. default is true.
*/
readText?: boolean
/**
* Enable grep logging to discover what files are searched.
*/
debug?: boolean
}
interface WorkspaceGrepResult {
files: WorkspaceFile[]
matches: WorkspaceFile[]
}
interface INIParseOptions extends JSONSchemaValidationOptions {
defaultValue?: any
}
interface FilterGitFilesOptions {
/**
* Ignore workspace .gitignore instructions
*/
applyGitIgnore?: false | undefined
}
interface FindFilesOptions extends FilterGitFilesOptions {
/** Glob patterns to ignore */
ignore?: ElementOrArray<string>
/**
* Set to false to skip read text content. True by default
*/
readText?: boolean
}
interface FileStats {
/**
* Size of the file in bytes
*/
size: number
mode: number
}
interface JSONSchemaValidationOptions {
schema?: JSONSchema
throwOnValidationError?: boolean
}
interface WorkspaceFileSystem {
/**
* Searches for files using the glob pattern and returns a list of files.
* Ignore `.env` files and apply `.gitignore` if present.
* @param glob
*/
findFiles(
glob: ElementOrArray<string>,
options?: FindFilesOptions
): Promise<WorkspaceFile[]>
/**
* Performs a grep search over the files in the workspace using ripgrep.
* @param pattern A string to match or a regex pattern.
* @param options Options for the grep search.
*/
grep(
pattern: string | RegExp,
options?: WorkspaceGrepOptions
): Promise<WorkspaceGrepResult>
grep(
pattern: string | RegExp,
glob: string,
options?: Omit<WorkspaceGrepOptions, "path" | "glob">
): Promise<WorkspaceGrepResult>
/**
* Reads metadata information about the file. Returns undefined if the file does not exist.
* @param filename
*/
stat(filename: string): Promise<FileStats>
/**
* Reads the content of a file as text
* @param path
*/
readText(path: string | Awaitable<WorkspaceFile>): Promise<WorkspaceFile>
/**
* Reads the content of a file and parses to JSON, using the JSON5 parser.
* @param path
*/
readJSON(
path: string | Awaitable<WorkspaceFile>,
options?: JSONSchemaValidationOptions
): Promise<any>
/**
* Reads the content of a file and parses to YAML.
* @param path
*/
readYAML(
path: string | Awaitable<WorkspaceFile>,
options?: JSONSchemaValidationOptions
): Promise<any>
/**
* Reads the content of a file and parses to XML, using the XML parser.
*/
readXML(
path: string | Awaitable<WorkspaceFile>,
options?: XMLParseOptions
): Promise<any>
/**
* Reads the content of a CSV file.
* @param path
*/
readCSV<T extends object>(
path: string | Awaitable<WorkspaceFile>,
options?: CSVParseOptions
): Promise<T[]>
/**
* Reads the content of a file and parses to INI
*/
readINI(
path: string | Awaitable<WorkspaceFile>,
options?: INIParseOptions
): Promise<any>
/**
* Reads the content of a file and attempts to parse it as data.
* @param path
* @param options
*/
readData(
path: string | Awaitable<WorkspaceFile>,
options?: CSVParseOptions &
INIParseOptions &
XMLParseOptions &
JSONSchemaValidationOptions
): Promise<any>
/**
* Appends text to a file as text to the file system. Creates the file if needed.
* @param path
* @param content
*/
appendText(path: string, content: string): Promise<void>
/**
* Writes a file as text to the file system
* @param path
* @param content
*/
writeText(path: string, content: string): Promise<void>
/**
* Caches a buffer to file and returns the unique file name
* @param bytes
*/
writeCached(
bytes: BufferLike,
options?: {
scope?: "workspace" | "run"
/**
* Filename extension
*/
ext?: string
}
): Promise<string>
/**
* Writes one or more files to the workspace
* @param file a in-memory file or list of files
*/
writeFiles(file: ElementOrArray<WorkspaceFile>): Promise<void>
/**
* Copies a file between two paths
* @param source
* @param destination
*/
copyFile(source: string, destination: string): Promise<void>
/**
* Opens a file-backed key-value cache for the given cache name.
* The cache is persisted across runs of the script. Entries are dropped when the cache grows too large.
* @param cacheName
*/
cache<K = any, V = any>(
cacheName: string
): Promise<WorkspaceFileCache<K, V>>
}
interface ToolCallContext {
log(message: string): void
debug(message: string): void
trace: ToolCallTrace
}
interface ToolCallback {
spec: ToolDefinition
options?: DefToolOptions
generator?: ChatGenerationContext
impl: (
args: { context: ToolCallContext } & Record<string, any>
) => Awaitable<ToolCallOutput>
}
interface ChatContentPartText {
/**
* The text content.
*/
text: string
/**
* The type of the content part.
*/
type: "text"
}
interface ChatContentPartImage {
image_url: {
/**
* Either a URL of the image or the base64 encoded image data.
*/
url: string
/**
* Specifies the detail level of the image. Learn more in the
* [Vision guide](https://platform.openai.com/docs/guides/vision#low-or-high-fidelity-image-understanding).
*/
detail?: "auto" | "low" | "high"
}
/**
* The type of the content part.
*/
type: "image_url"
}
interface ChatContentPartInputAudio {
input_audio: {
/**
* Base64 encoded audio data.
*/
data: string
/**
* The format of the encoded audio data. Currently supports "wav" and "mp3".
*/
format: "wav" | "mp3"
}
/**
* The type of the content part. Always `input_audio`.
*/
type: "input_audio"
}
interface ChatContentPartFile {
file: {
/**
* The base64 encoded file data, used when passing the file to the model as a
* string.
*/
file_data?: string
/**
* The ID of an uploaded file to use as input.
*/
file_id?: string
/**
* The name of the file, used when passing the file to the model as a string.
*/
filename?: string
}
/**
* The type of the content part. Always `file`.
*/
type: "file"
}
interface ChatContentPartRefusal {
/**
* The refusal message generated by the model.
*/
refusal: string
/**
* The type of the content part.
*/
type: "refusal"
}
interface ChatSystemMessage {
/**
* The contents of the system message.
*/
content: string | ChatContentPartText[]
/**
* The role of the messages author, in this case `system`.
*/
role: "system"
/**
* An optional name for the participant. Provides the model information to
* differentiate