life/dist/exports/server.mjs

Life.js is the first fullstack framework to build agentic web applications. It is minimal, extensible, and typesafe. Well, everything you love.

import {
  RollingBuffer
} from "../chunk-LPKEDFLM.mjs";
import {
  audioChunkToMs,
  defineConfig
} from "../chunk-XNMZQAOR.mjs";
import {
  internalEventsDef,
  toMethodName
} from "../chunk-X455LF5V.mjs";
import {
  AsyncQueue,
  deepClone,
  zodObjectWithTelemetry
} from "../chunk-D2T23PCX.mjs";
import {
  attempt,
  dataOrThrow,
  failure,
  lifeError,
  newId,
  success
} from "../chunk-ZHBK6UTM.mjs";
import {
  __name
} from "../chunk-2D3UJWOA.mjs";

// agent/server/builder.ts
import { z } from "zod";
var AgentBuilder = class _AgentBuilder {
  static {
    __name(this, "AgentBuilder");
  }
  def;
  constructor(definition) {
    this.def = definition;
  }
  config(config) {
    const builder = new _AgentBuilder({ ...this.def, config });
    const builderWithPlugins = _AgentBuilder.withPluginsMethods(builder);
    return builderWithPlugins;
  }
  scope(scope) {
    const builder = new _AgentBuilder({ ...this.def, scope });
    const builderWithPlugins = _AgentBuilder.withPluginsMethods(builder);
    return builderWithPlugins;
  }
  /**
   * Register plugins to extend the agent server features.
   * Defaults to `[generation, memories, stores, actions, percepts]` plugins if not specified.
   *
   * In case you want to register custom plugins and still keep the defaults you can do:
   * ```ts
   * import { defaults } from "life/client";
   *
   * defineAgentClient("my-agent").plugins([...defaults.plugins, myCustomPlugin]);
   * ```
   *
   * Or if you want only some of the defaults, you can do:
   * ```ts
   * import { defaults } from "life/client";
   *
   * defineAgentClient("my-agent").plugins([defaults.plugins.generation, defaults.plugins.memories]);
   * ```
   */
  plugins(plugins) {
    const builder = new _AgentBuilder({ ...this.def, plugins: plugins.map((p) => p.def) });
    const builderWithPlugins = _AgentBuilder.withPluginsMethods(builder);
    return builderWithPlugins;
  }
  // biome-ignore lint/suspicious/noExplicitAny: reason
  static withPluginsMethods(builder) {
    for (const plugin of builder.def.plugins) {
      Object.assign(builder, {
        [toMethodName(plugin.name)]: (config) => {
          const newBuilder = new _AgentBuilder({
            ...builder.def,
            pluginConfigs: {
              ...builder.def.pluginConfigs ?? {},
              [plugin.name]: config
            }
          });
          return _AgentBuilder.withPluginsMethods(newBuilder);
        }
      });
    }
    return builder;
  }
};
function defineAgent(name) {
  const defaultDefinition = {
    name,
    config: {},
    scope: { schema: z.object(), hasAccess: /* @__PURE__ */ __name(() => true, "hasAccess") },
    plugins: [...defaults.plugins].map((p) => p.def),
    pluginConfigs: {}
  };
  const builder = new AgentBuilder(defaultDefinition);
  const builderWithPlugins = AgentBuilder.withPluginsMethods(builder);
  return builderWithPlugins;
}
__name(defineAgent, "defineAgent");

// plugins/defaults/memories/server/builder.ts
import { z as z2 } from "zod";
var memoryConfigSchema = z2.object({
  behavior: z2.enum(["blocking", "non-blocking"]).prefault("blocking")
});
var MemoryDefinitionBuilder = class _MemoryDefinitionBuilder {
  static {
    __name(this, "MemoryDefinitionBuilder");
  }
  _definition;
  constructor(def) {
    this._definition = def;
  }
  dependencies(dependencies) {
    return new _MemoryDefinitionBuilder({
      ...this._definition,
      dependencies: "_definition" in dependencies ? dependencies._definition : dependencies
    });
  }
  config(config) {
    const parsedConfig = memoryConfigSchema.parse(config);
    return new _MemoryDefinitionBuilder({
      ...this._definition,
      config: parsedConfig
    });
  }
  output(params) {
    return new _MemoryDefinitionBuilder({
      ...this._definition,
      output: params
    });
  }
  // biome-ignore lint/nursery/noShadow: expected here
  onHistoryChange(params) {
    return new _MemoryDefinitionBuilder({
      ...this._definition,
      onHistoryChange: params
    });
  }
};
function defineMemory(name) {
  return new MemoryDefinitionBuilder({
    name,
    config: memoryConfigSchema.parse({}),
    // Will default to { behavior: "blocking" }
    dependencies: {
      stores: [],
      collections: []
    }
  });
}
__name(defineMemory, "defineMemory");

// plugins/defaults/stores/builder.ts
import { z as z3 } from "zod";
var commonStoreConfigSchema = z3.object({
  schema: z3.custom((val) => {
    if (val instanceof z3.ZodArray || val instanceof z3.ZodRecord) return true;
    return false;
  })
});
var controlledStoreConfigSchema = commonStoreConfigSchema.extend({
  type: z3.literal("controlled"),
  ttl: z3.number().optional()
});
var freeformStoreConfigSchema = commonStoreConfigSchema.extend({
  type: z3.literal("freeform")
});
var storeConfigSchema = z3.union([controlledStoreConfigSchema, freeformStoreConfigSchema]);
var StoreDefinitionBuilder = class _StoreDefinitionBuilder {
  static {
    __name(this, "StoreDefinitionBuilder");
  }
  _definition;
  constructor(def) {
    this._definition = def;
  }
  config(config) {
    const parsedConfig = storeConfigSchema.parse(config);
    return new _StoreDefinitionBuilder({
      ...this._definition,
      config: parsedConfig
    });
  }
  retrieve(retrieve) {
    return new _StoreDefinitionBuilder({
      ...this._definition,
      retrieve
    });
  }
};
function defineStore(name) {
  return new StoreDefinitionBuilder({ name, config: storeConfigSchema.parse({}) });
}
__name(defineStore, "defineStore");

// plugins/server/builder.ts
import { z as z4 } from "zod";
var definePluginConfig = /* @__PURE__ */ __name((definition) => zodObjectWithTelemetry(definition), "definePluginConfig");
var definePluginContext = /* @__PURE__ */ __name((definition) => zodObjectWithTelemetry(definition), "definePluginContext");
var definePluginEvents = /* @__PURE__ */ __name((...events) => [...internalEventsDef, ...events], "definePluginEvents");
var PluginBuilder = class _PluginBuilder {
  static {
    __name(this, "PluginBuilder");
  }
  def;
  constructor(definition) {
    this.def = definition;
  }
  /**
   * ### `.dependencies()`
   *
   * Specify other plugins as required by this plugin.
   *
   * Their context, events, and config can then be accessed from `dependencies.*` inside handlers.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .dependencies([anotherPlugin]);
   *    .addHandler({
   *        name: "handler-1",
   *        mode: "block",
   *        onEvent: ({ dependencies }) => {
   *            if (dependencies.anotherPlugin.config.delayMs > 10) { // <-- Here
   *              // ...
   *            }
   *        },
   *    });
   * ```
   * &nbsp;
   *
   * ---
   * @param plugins - The dependencies definitions.
   * @returns TypedPluginBuilder
   */
  dependencies(plugins) {
    const dependencies = plugins.map((p) => p.def);
    const builder = new _PluginBuilder({ ...this.def, dependencies });
    return builder;
  }
  /**
   * ### `.config()`
   *
   * Add a configuration that users can provide to tweak plugin's behavior.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const myPlugin = definePlugin("my-plugin")
   *    .config({ schema: z.object({ enableVoice: z.boolean() }) });
   *
   * const myAgent = defineAgent("my-agent")
   *    .plugins([myPlugin])
   *    .myPlugin({ enableVoice: true }); // <-- Here
   * ```
   * &nbsp;
   *
   * ---
   *
   * The provided config can then be accessed from `plugin.config` inside handlers.
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .config({ schema: z.object({ enableVoice: z.boolean() }) });
   *    .addHandler({
   *      name: "handler-1",
   *      mode: "block",
   *      onEvent: ({ plugin }) => {
   *        if (plugin.config.enableVoice) { // <-- Here
   *          // ...
   *        }
   *      },
   *    });
   *
   * ```
   * &nbsp;
   *
   * ---
   * @param config - The config definition.
   * @returns TypedPluginBuilder
   */
  config(config) {
    return this.$config(zodObjectWithTelemetry(config));
  }
  /**
   * ### `.$config()`
   *
   * Define plugin config from the output of `definePluginConfig()`.
   *
   * @see TODO: Add docs link for `definePluginConfig()`
   *
   * ---
   * @param config - The config definition.
   * @returns TypedPluginBuilder
   */
  $config(config) {
    const builder = new _PluginBuilder({ ...this.def, config });
    return builder;
  }
  /**
   * ### `.context()`
   *
   * Add an in-memory object that handlers can read/write to share data without race conditions.
   *
   * Context is also exposed to dependent plugins and synced to the frontend via RPC.
   *
   * The context must be serializable, and thus cannot contain functions, instances, etc.
   * Use `state` property in handlers for non-serializable data instead.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .context(z.object({ name: z.string() }));
   * ```
   * &nbsp;
   *
   * ---
   * @param context - The context definition.
   */
  context(context) {
    return this.$context(zodObjectWithTelemetry(context));
  }
  /**
   * ### `.$context()`
   *
   * Define plugin context from the output of `definePluginContext()`.
   *
   * @see TODO: Add docs link for `definePluginContext()`
   *
   * ---
   * @param context - The context definition.
   * @returns TypedPluginBuilder
   */
  $context(context) {
    const builder = new _PluginBuilder({ ...this.def, context });
    return builder;
  }
  /**
   * ### `.events()`
   *
   * Add events that handlers can emit and observe to communicate with each other.
   *
   * Each plugin has a single event queue enforcing the order of execution, making
   * plugins more predictable and easier to write and maintain.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .events([{ name: "my-event" }])
   *    .addHandler({
   *      name: "handler-1",
   *      mode: "block",
   *      onEvent: ({ event, plugin }) => {
   *        // Emit 'my-event' when the plugin starts
   *        if (event.name === "plugin.start") {
   *          plugin.events.emit({ name: "my-event" });
   *        }
   *      },
   *    })
   *    .addHandler({
   *      name: "handler-2",
   *      mode: "block",
   *      onEvent: ({ event, plugin }) => {
   *        // Listen to 'my-event' from handler-1
   *        if (event.name === "my-event") {
   *          // Do something...
   *        }
   *      },
   *    });
   * ```
   * &nbsp;
   *
   * ---
   * @param events - The events definition.
   * @returns TypedPluginBuilder
   */
  events(...events) {
    return this.$events([...internalEventsDef, ...events]);
  }
  /**
   * ### `.$events()`
   *
   * Define plugin events from the output of `definePluginEvents()`.
   *
   * @see TODO: Add docs link for `definePluginEvents()`
   *
   * ---
   * @param events - The events definition.
   * @returns TypedPluginBuilder
   */
  $events(events) {
    const builder = new _PluginBuilder({ ...this.def, events });
    return builder;
  }
  /**
   * ### `.addHandler()`
   *
   * Add a handler function to react to events from this plugin or dependent plugins.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .addHandler({
   *      name: "handler-1",
   *      mode: "block",
   *      onEvent: ({ event, plugin }) => {
   *        // Do something...
   *      },
   *    });
   * ```
   * &nbsp;
   *
   * ---
   *
   * Handlers can return a value, which can be retrieved using the `events.waitForResult(eventId, handlerName)` method.
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .events([{ name: "my-event" }])
   *    .addHandler({
   *      name: "handler-1",
   *      mode: "block",
   *      onEvent: () => 123;
   *    })
   *    .addHandler({
   *      name: "handler-2",
   *      mode: "block",
   *      onEvent: ({ event, plugin }) => {
   *        if (event.name === "my-event") {
   *          const eventId = plugin.events.emit({ name: "my-event" });
   *          const result = await plugin.events.waitForResult(eventId, "handler-1");
   *          // result: string (type-safe!)
   *        }
   *      },
   *    })
   * ```
   * &nbsp;
   *
   * ---
   * @param handler - The handler definition.
   * @returns TypedPluginBuilder
   */
  addHandler(handler) {
    const handlers = [...this.def.handlers ?? [], handler];
    const builder = new _PluginBuilder({ ...this.def, handlers });
    return builder;
  }
  /**
   * ### `.removeHandler()`
   *
   * Remove a previously added handler from the plugin.
   *
   * @see TODO: Add docs link
   *
   * @example
   * ```ts
   * const plugin = definePlugin("my-plugin")
   *    .addHandler({
   *      name: "handler-1",
   *      mode: "block",
   *      onEvent: ({ event, plugin }) => void 0,
   *    });
   *
   * const pluginWithoutHandler1 = plugin.removeHandler("handler-1");
   * ```
   * &nbsp;
   *
   * ---
   * @param name - The name of the handler to remove.
   * @returns TypedPluginBuilder
   */
  removeHandler(name) {
    const handlers = this.def.handlers.filter((h) => h.name !== name);
    const builder = new _PluginBuilder({ ...this.def, handlers });
    return builder;
  }
};
function definePlugin(name) {
  return new PluginBuilder({
    name,
    dependencies: [],
    config: zodObjectWithTelemetry({ schema: z4.object() }),
    context: zodObjectWithTelemetry({ schema: z4.object() }),
    events: internalEventsDef,
    handlers: []
  });
}
__name(definePlugin, "definePlugin");

// shared/messages.ts
import { z as z5 } from "zod";
var baseMessageSchema = {
  id: z5.string(),
  createdAt: z5.number(),
  lastUpdated: z5.number()
};
var userMessageSchema = z5.object({
  ...baseMessageSchema,
  role: z5.literal("user"),
  content: z5.string().prefault("")
});
var systemMessageSchema = z5.object({
  ...baseMessageSchema,
  role: z5.literal("system"),
  content: z5.string().prefault("")
});
var agentToolRequestSchema = z5.object({
  toolRequestId: z5.string(),
  toolName: z5.string(),
  toolInput: z5.record(z5.string(), z5.any())
});
var agentMessageSchema = z5.object({
  ...baseMessageSchema,
  role: z5.literal("agent"),
  content: z5.string().prefault(""),
  toolsRequests: z5.array(agentToolRequestSchema).prefault([])
});
var toolResponseMessageSchema = z5.object({
  ...baseMessageSchema,
  role: z5.literal("tool"),
  toolRequestId: z5.string(),
  toolName: z5.string(),
  toolSuccess: z5.boolean(),
  toolOutput: z5.record(z5.string(), z5.any()).optional(),
  toolError: z5.string().optional()
});
var messageSchema = z5.discriminatedUnion("role", [
  userMessageSchema,
  systemMessageSchema,
  agentMessageSchema,
  toolResponseMessageSchema
]);
var createOmitFields = { createdAt: true, lastUpdated: true, id: true };
var createMessageInputSchema = z5.discriminatedUnion("role", [
  userMessageSchema.omit(createOmitFields),
  systemMessageSchema.omit(createOmitFields),
  agentMessageSchema.omit(createOmitFields),
  toolResponseMessageSchema.omit(createOmitFields)
]);
var updateOmitFields = { createdAt: true, lastUpdated: true, id: true };
var updateMessageInputSchema = z5.discriminatedUnion("role", [
  userMessageSchema.omit(updateOmitFields).partial().extend({ role: z5.literal("user") }),
  systemMessageSchema.omit(updateOmitFields).partial().extend({ role: z5.literal("system") }),
  agentMessageSchema.omit(updateOmitFields).partial().extend({ role: z5.literal("agent") }),
  toolResponseMessageSchema.omit(updateOmitFields).partial().extend({ role: z5.literal("tool") })
]);
var MessageList = class {
  static {
    __name(this, "MessageList");
  }
  #messages = [];
  constructor(messages) {
    this.#messages = deepClone(messages ?? []);
  }
  getAll() {
    return attempt(() => deepClone(this.#messages));
  }
  get(id) {
    const [err, messages] = this.getAll();
    if (err) return failure(err);
    return success(messages.find((message) => message.id === id));
  }
  findLastFromRoles(roles) {
    const [err, messages] = this.getAll();
    if (err) return failure(err);
    const lastMessage = messages.reverse().find((message) => roles.includes(message.role));
    return success(lastMessage);
  }
  create(message) {
    const { data: validatedMessage, error: validatedMessageError } = createMessageInputSchema.safeParse(message);
    if (validatedMessageError)
      return failure({
        code: "Validation",
        message: "Invalid message shape.",
        cause: validatedMessageError
      });
    const newMessage = {
      ...validatedMessage,
      id: newId("message"),
      createdAt: Date.now(),
      lastUpdated: Date.now()
    };
    this.#messages.push(newMessage);
    return success(newMessage.id);
  }
  update(id, role, message) {
    const { data: validatedMessage, error: validationError } = updateMessageInputSchema.safeParse({
      ...message,
      role
    });
    if (validationError)
      return failure({
        code: "Validation",
        message: "Invalid message shape.",
        cause: validationError
      });
    const [err, existingMessage] = this.get(id);
    if (err) return failure(err);
    if (!existingMessage)
      return failure({
        code: "NotFound",
        message: `Message with id '${id}' does not exist.`
      });
    if (existingMessage.role !== role)
      return failure({
        code: "Validation",
        message: "Invalid message role provided.",
        cause: new Error(`Message with id '${id}' is not a ${role} message.`)
      });
    const newMessage = {
      ...existingMessage,
      ...validatedMessage,
      lastUpdated: Date.now()
    };
    this.#messages = this.#messages.map((m) => m.id === id ? newMessage : m);
    return success(id);
  }
};

// plugins/defaults/generation/server/config.ts
import z6 from "zod";
var generationPluginConfigDef = definePluginConfig({
  schema: z6.object({
    voiceDetection: z6.object({
      /**
       * VAD score threshold for detecting voice activity during speech (0.0-1.0).
       * Uses hysteresis: once voice is detected, this higher threshold must be exceeded
       * to continue considering audio as speech. This prevents flickering between
       * speech/silence states during continuous speaking.
       */
      scoreInThreshold: z6.number().prefault(0.5),
      /**
       * VAD score threshold for detecting end of voice activity (0.0-1.0).
       * When voice is active, audio must fall below this lower threshold to be
       * considered silence. Lower than `scoreInThreshold` to provide hysteresis.
       */
      scoreOutThreshold: z6.number().prefault(0.25),
      /**
       * Number of silent audio chunks to buffer before voice starts (default: 100 ≈ 1s).
       * These chunks are emitted when voice is detected, ensuring the first syllables
       * and word onsets are captured for better STT accuracy.
       *
       * Can slightly impact STT latency, as increases the amount of audio to be processed.
       */
      prePaddingChunks: z6.number().prefault(100),
      /**
       * Number of silent chunks to append after voice ends before finalizing (default: 200 ≈ 2s).
       *
       * Most STT providers require silence padding to finalize transcription. For example,
       * Deepgram with `endpointing: 0` and `no_delay: true` still needs substantial
       * silence to return results. Too few chunks may cause the STT to hang indefinitely.
       *
       * This default (200) balances latency and stability across providers. If your STT
       * finalizes transcripts quickly, consider lowering this value. Benchmark carefully.
       */
      postPaddingChunks: z6.number().prefault(200),
      /**
       * Minimum duration (in ms) of continuous voice to trigger agent interruption.
       * Uses a sliding window to accumulate voice chunks and filter out VAD false positives.
       * Only when accumulated voice duration exceeds this threshold, the agent will be interrupted.
       */
      minVoiceInterruptionMs: z6.number().prefault(50)
    }).prefault({}),
    endOfTurnDetection: z6.object({
      /**
       * Probability threshold (0.0-1.0) that determines when the user has finished speaking.
       * If EOU model confidence >= threshold, agent responds immediately. Otherwise, waits
       * with an adaptive timeout (see min/maxTimeoutMs below).
       *
       * Tuning considerations:
       * - Too low: Agent may interrupt users mid-sentence, creating awkward overlaps
       * - Too high: Agent waits longer before responding, increasing perceived latency
       * - Default (0.6): Balanced trade-off between responsiveness and avoiding interruptions
       */
      threshold: z6.number().prefault(0.6),
      /**
       * Fallback timeout (in ms) ensuring the agent eventually responds even when EOU
       * confidence stays low. Prevents the agent from waiting indefinitely if the model
       * never reaches the threshold (e.g., incomplete sentences, uncertain patterns).
       */
      minTimeoutMs: z6.number().prefault(300),
      /**
       * Maximum wait time (in ms) when EOU confidence is at or near zero. As confidence
       * increases, the timeout shrinks adaptively toward minTimeoutMs using:
       * `timeout = max(minTimeoutMs, maxTimeoutMs * (1 - probability / threshold))`
       *
       * This creates natural turn-taking: high confidence = quick response, low confidence = patient waiting.
       */
      maxTimeoutMs: z6.number().prefault(5e3)
    }).prefault({})
  })
});

// plugins/defaults/generation/server/context.ts
import z8 from "zod";

// plugins/defaults/generation/server/status.ts
import z7 from "zod";
var generationStatusSchema = z7.object({
  listening: z7.boolean().prefault(true),
  thinking: z7.boolean().prefault(false),
  speaking: z7.boolean().prefault(false)
}).prefault({});
var computeStatus = /* @__PURE__ */ __name((oldStatus, eventType) => {
  try {
    if (eventType === "agent.thinking-start") {
      return success({ ...oldStatus, listening: false, thinking: true });
    } else if (eventType === "agent.thinking-end") {
      return success({ ...oldStatus, thinking: false });
    } else if (eventType === "agent.speaking-end") {
      return success({ ...oldStatus, listening: true, thinking: false, speaking: false });
    } else if (eventType === "agent.speaking-start") {
      return success({ ...oldStatus, listening: false, speaking: true });
    }
    return success(oldStatus);
  } catch (error) {
    return failure({ code: "Unknown", cause: error });
  }
}, "computeStatus");

// plugins/defaults/generation/server/context.ts
var generationPluginContextDef = definePluginContext({
  schema: z8.object({
    /**
     * The entire history of messages handled by the generation plugin.
     */
    messages: z8.array(messageSchema).prefault([
      {
        id: "system-1",
        createdAt: Date.now(),
        lastUpdated: Date.now(),
        role: "system",
        content: `        
# General Instructions
The below instructions define your general behavior and attitude as an agent.
Further down, you'll find more specific instructions about your role, your context, and your capabilities.

## Attitude
The user is speaking to you using their voice and/or by typing on a chat interface (if available).
Your answers are streamed back to the user as text and/or voice as well. 

As a multimodal agent, your answers must sound natural when converted to voice, but also when read as text.
Here's how to produce **spoken language**:

- Use a natural, informal, conversational vocabulary and tone. You're original, have your own personality.
  Example: "Yeah I think I got your point, but are you sure this would work for all configurations?"

- Include filler words and hesitations (e.g., "um", "uh", "like", "oh", "ah") where appropriate.
  Example: "Um, let me think... maybe we could try that approach?"
  
- Embrace incomplete sentences or broken thoughts to simulate true speech patterns.
  Example: "Maybe we should... oh wait, here's another idea..."

- If a user asks something like "can you hear me?", say yes, you can hear them.

- Remember that the user can interrupt you at any time, handle that gracefully.

- Show traits of emotional intelligence like self-awareness, empathy, humor, excitment.

- Employ small-talks, both at the beginning and during the conversation, as long as it doesn't disrupt the conversation flow.

- Relate to the user's answers, so they feel heard. Don't repeat their entire answers, but just small notes and summaries showing that you're listening and keeping track of the conversation.

- Tend to mirror the user's expression, words, etc. It is proven that it increases the rapport.

- Overall, be positive, open and warm to the users ideas, encourage them to talk more about themselves.

## Markdown
Your answers can include Markdown following the CommonMark, GFM, LaTex, and Mermaid specifications.

Before being converted to voice with a TTS model, your answers will be pre-processed as follows:

- Markdown symbols will be stripped out, e.g., "**bold**" becomes "bold"

- The following blocks will be entirely ignored from the TTS speech: table, code blocks, LaTeX blocks and Mermaid blocks.
  So if you need to include comment about such a block, do it right before or after the block.
  Example: 
  \`\`\`markdown
  You were right, there is a **huge** spike in February!
  | Month | Value |
  |-------|-------|
  | January | 100   |
  | February | 1380 (huge spike)   |
  | March | 300   |
  | April | 350   |
  | May | 200   |
  
  Do you want me to investigate further?
  \`\`\`
  will be converted as voice as "You were right, there is a huge spike in February! Do you want me to investigate further?"
  with a short pause in speech when the table is being rendered.

- While LaTeX blocks are excluded from speech, inline LaTeX is converted into spoken form.
  Example: "$x^2$" becomes "x squared", "$y = x^2 + 1$" becomes "y equals x squared plus one".
  Avoid using long inline LaTeX, only simple and short formulas or variable names. Use LaTeX blocks for the rest.

- While code blocks are excluded from speech, inline code is converted into plain text with symbol stripped out.
  Example: "Call the \`generate()\` function" becomes "Call the generate function".
  Avoid using long inline code, only simple and short code snippets, like variables or functions names. Use code blocks for the rest.

- Links, images, and their references are converted into just the alt or title text when provided.
  Example: "[link](https://example.com)" becomes "link", "![image](https://example.com/image.png)" becomes "image".
  If an alt of title is not provided, the image or link won't appear in the speech.
  Example: "Can you check https://example.com ?" becomes "Can you check ?" which might be confusing.

- List are converted into plain text, without delimiters symbols, even for ordered lists. So make your list items content sounding natural when read.
  Example:
  "Here are the steps to follow:
  1. First, open the app.
  2. Then, click on the "+" button.
  3. Finally, reload the page.
  "
  will be converted as voice as "Here are the steps to follow: First, open the app. Then, click on the plus button. Finally, reload the page."

So for ordered lists for example, when numbering matters, 

## Bug / Error handling
If something doesn't work as expected, retry with the user 2 times, then if it still doesn't work:
- Notify the developers by sending them a message using the "notify_developers" tool.
- Then only, explain to the user that there might be some temporary issue right now, that you've notified the developers, and ask the user to try again later.
- Don't bother the user with too many retry back and forth, if you see that it's not working after two retries, just notify the devs and move on.
        `
      }
    ]),
    /**
     * The current generation status.
     * Contains { listening, thinking, speaking } flags.
     */
    status: generationStatusSchema,
    /**
     * Whether the generation plugin should generate and stream voice back
     * to the user. If true, solely text chunks will be emitted.
     */
    voiceEnabled: z8.boolean().prefault(true)
  })
});

// plugins/defaults/generation/server/events.ts
import z10 from "zod";

// models/llm/resources.ts
import z9 from "zod";
var llmToolSchema = z9.object({
  name: z9.string(),
  description: z9.string(),
  schema: z9.object({
    input: z9.instanceof(z9.ZodObject),
    output: z9.instanceof(z9.ZodObject)
  }),
  run: z9.function({
    input: [z9.record(z9.string(), z9.any())],
    output: z9.union([
      z9.object({
        success: z9.boolean(),
        output: z9.record(z9.string(), z9.any()).optional(),
        error: z9.string().optional()
      }),
      z9.promise(
        z9.object({
          success: z9.boolean(),
          output: z9.record(z9.string(), z9.any()).optional(),
          error: z9.string().optional()
        })
      )
    ])
  })
});
var llmResourcesSchema = z9.object({
  messages: z9.array(messageSchema),
  tools: z9.array(llmToolSchema)
});

// plugins/defaults/generation/server/events.ts
var insertEventBaseSchema = z10.object({
  interrupt: z10.enum(["abrupt", "smooth"]).or(z10.literal(false)).prefault("abrupt"),
  preventInterruption: z10.boolean().prefault(false)
});
var generationPluginEventsDef = definePluginEvents(
  { name: "messages.create", dataSchema: z10.object({ message: createMessageInputSchema }) },
  {
    name: "messages.update",
    dataSchema: z10.object({
      id: z10.string(),
      role: z10.enum(["user", "system", "agent", "tool"]),
      message: createMessageInputSchema
    })
  },
  { name: "user.audio-chunk", dataSchema: z10.object({ audioChunk: z10.custom() }) },
  { name: "user.voice-start" },
  {
    name: "user.voice-chunk",
    dataSchema: z10.discriminatedUnion("type", [
      z10.object({ type: z10.literal("voice"), voiceChunk: z10.custom() }),
      z10.object({
        type: z10.literal("padding"),
        voiceChunk: z10.custom(),
        paddingSide: z10.enum(["pre", "post"]),
        paddingIndex: z10.number()
      })
    ])
  },
  { name: "user.voice-end" },
  { name: "user.text-chunk", dataSchema: z10.object({ textChunk: z10.string() }) },
  { name: "user.interrupted" },
  { name: "agent.thinking-start" },
  { name: "agent.thinking-end" },
  { name: "agent.speaking-start" },
  { name: "agent.speaking-end" },
  { name: "agent.continue", dataSchema: insertEventBaseSchema },
  {
    name: "agent.decide",
    dataSchema: insertEventBaseSchema.extend({ messages: z10.array(messageSchema) })
  },
  {
    name: "agent.say",
    dataSchema: insertEventBaseSchema.extend({ text: z10.string() })
  },
  {
    name: "agent.interrupt",
    dataSchema: z10.object({
      reason: z10.string(),
      author: z10.enum(["user", "application"]),
      force: z10.boolean().prefault(false)
    })
  },
  { name: "agent.resources-request" },
  {
    name: "agent.resources-response",
    dataSchema: z10.object({ requestId: z10.string(), resources: llmResourcesSchema })
  },
  {
    name: "agent.tool-requests",
    dataSchema: z10.object({ requests: z10.array(agentToolRequestSchema) })
  },
  {
    name: "agent.interrupted",
    dataSchema: z10.object({
      reason: z10.string(),
      forced: z10.boolean(),
      author: z10.enum(["user", "application"])
    })
  },
  { name: "agent.text-chunk", dataSchema: z10.object({ textChunk: z10.string() }) },
  { name: "agent.voice-chunk", dataSchema: z10.object({ voiceChunk: z10.custom() }) }
);

// plugins/defaults/generation/server/orchestrator.ts
import { z as z11 } from "zod";

// plugins/defaults/generation/server/generation.ts
var Generation = class {
  static {
    __name(this, "Generation");
  }
  id = newId("generation");
  queue = new AsyncQueue();
  progress = "idle";
  params = { prefix: "", needContinue: false, preventInterruption: false };
  #agent;
  #voiceEnabled;
  #llmJob = null;
  #ttsJob = null;
  #toolRequests = null;
  #statusChangeCallbacks = [];
  constructor(params) {
    this.#agent = params.agent;
    this.#voiceEnabled = params.voiceEnabled;
  }
  onStatusChange(callback) {
    this.#statusChangeCallbacks.push(callback);
  }
  canBeInterrupted() {
    return !this.params.preventInterruption && (this.progress === "started" || this.queue.length() > 0);
  }
  canStart() {
    return this.progress === "idle" && (this.params.prefix || this.params.needContinue);
  }
  addInsertEvent(event) {
    if (this.progress !== "idle")
      throw new Error("Cannot add continue/say operation when not idle or waiting.");
    if (event.name === "agent.continue") this.params.needContinue = true;
    else if (event.name === "agent.say") this.params.prefix += event.data.text;
    if (!this.params.preventInterruption)
      this.params.preventInterruption = event.data.preventInterruption ?? false;
  }
  async start(resources) {
    if (this.progress !== "idle") throw new Error("Cannot start generation when not idle.");
    this.progress = "started";
    if (this.#voiceEnabled) {
      this.#ttsJob = await this.#agent.models.tts.generate();
      this.#startTTS();
    }
    this.#startLLM(resources);
    for (const callback of this.#statusChangeCallbacks) callback("started");
  }
  async #startTTS() {
    if (!this.#ttsJob) throw new Error("TTS job not initialized, should not happen.");
    for await (const chunk of this.#ttsJob.stream) {
      if (chunk.type === "content")
        this.queue.push({
          type: "content",
          textChunk: chunk.textChunk,
          voiceChunk: chunk.voiceChunk
        });
      else if (chunk.type === "end") {
        if (this.#toolRequests) {
          this.queue.push({ type: "tool-requests", requests: this.#toolRequests });
        }
        this.end();
        break;
      } else if (chunk.type === "error") console.error("TTS error", JSON.stringify(chunk));
    }
  }
  async #startLLM(resources) {
    if (this.params.prefix) {
      if (this.#voiceEnabled) await this.#ttsJob?.inputText(this.params.prefix);
      else this.queue.push({ type: "content", textChunk: this.params.prefix });
    }
    if (!this.params.needContinue) {
      if (this.#voiceEnabled) await this.#ttsJob?.inputText("", true);
      else this.end();
      return;
    }
    if (!resources) throw new Error("Resources are required to continue LLM generation.");
    this.#llmJob = await this.#agent.models.llm.generateMessage(resources);
    let hasContent = false;
    for await (const chunk of this.#llmJob.stream) {
      if (chunk.type === "error") console.error("LLM error", chunk);
      else if (this.#voiceEnabled) {
        if (chunk.type === "content") {
          await this.#ttsJob?.inputText(chunk.content);
          hasContent = true;
        } else if (chunk.type === "tools") {
          if (hasContent) this.#toolRequests = chunk.tools;
          else {
            this.queue.push({ type: "tool-requests", requests: chunk.tools });
            break;
          }
        } else if (chunk.type === "end") {
          await this.#ttsJob?.inputText("", true);
          break;
        }
      } else if (chunk.type === "content")
        this.queue.push({ type: "content", textChunk: chunk.content });
      else if (chunk.type === "tools") {
        this.queue.push({ type: "tool-requests", requests: chunk.tools });
        this.end();
        break;
      } else if (chunk.type === "end") {
        this.end();
        break;
      }
    }
  }
  // Called by the orchestrator when end chunks are consumed
  end(early = false) {
    if (this.#llmJob) this.#llmJob.cancel();
    if (this.#ttsJob) this.#ttsJob.cancel();
    if (early) this.queue.pushFirst({ type: "end" });
    else this.queue.push({ type: "end" });
    this.progress = "ended";
    for (const callback of this.#statusChangeCallbacks) callback("ended");
  }
};

// plugins/defaults/generation/server/orchestrator.ts
var GenerationOrchestrator = class {
  static {
    __name(this, "GenerationOrchestrator");
  }
  #agent;
  #plugin;
  #eventsQueue = new AsyncQueue();
  #generationsQueue = new AsyncQueue();
  #generations = [];
  #decidePromises = [];
  #generationsResourcesRequestsIds = {};
  #resourcesResponses = {};
  constructor(params) {
    this.#agent = params.agent;
    this.#plugin = params.plugin;
  }
  pushEvent(event) {
    this.#eventsQueue.push(event);
  }
  async start() {
    this.#consumeGenerations();
    for await (const event of this.#eventsQueue) {
      if (this.#isGenerationEvent(event)) await this.#processGenerationEvent(event);
    }
  }
  #createGeneration() {
    const generation = new Generation({
      agent: this.#agent,
      voiceEnabled: this.#plugin.context.get().voiceEnabled
    });
    this.#generations.push(generation);
    generation.onStatusChange(() => {
      const runningCount = this.#generations.filter((g) => g.progress === "started").length;
      if (this.#plugin.context.get().status.thinking) {
        if (runningCount === 0)
          this.#plugin.events.emit({ name: "agent.thinking-end", urgent: true });
      } else if (runningCount > 0)
        this.#plugin.events.emit({ name: "agent.thinking-start", urgent: true });
    });
    return generation;
  }
  async #processGenerationEvent(event) {
    let generation = this.#generations.find((g) => g.progress === "idle");
    if (!generation) generation = await this.#createGeneration();
    if (event.name === "agent.continue") this.#processInsertEvent(generation, event);
    else if (event.name === "agent.say") this.#processInsertEvent(generation, event);
    else if (event.name === "agent.decide") this.#processDecideEvent(generation, event);
    else if (event.name === "agent.interrupt") this.#processInterruptEvent(event);
    else if (event.name === "agent.resources-response") this.#processResourcesResponseEvent(event);
    if (!generation.canStart()) return;
    if (this.#isQueueBusy()) return;
    if (event.name === "agent.continue") {
      const requestId = this.#plugin.events.emit({ name: "agent.resources-request" });
      this.#generationsResourcesRequestsIds[generation.id] = requestId;
    }
    if (generation.params.needContinue && !((this.#generationsResourcesRequestsIds[generation.id] ?? "") in this.#resourcesResponses))
      return;
    const runningGeneration = this.#generations.find((g) => g.progress !== "idle");
    if (runningGeneration) {
    } else {
      const resourceRequestId = this.#generationsResourcesRequestsIds[generation.id];
      const resources = resourceRequestId ? this.#resourcesResponses[resourceRequestId] : void 0;
      generation.start(resources);
      this.#generationsQueue.push(generation);
    }
  }
  #processInterruptEvent(event) {
    let interrupted = false;
    for (const generation of this.#generations) {
      if (generation.canBeInterrupted() || event.data.force) {
        generation.end(true);
        if (generation.queue.totalLength() > generation.queue.length()) interrupted = true;
      }
    }
    if (interrupted) {
      this.#plugin.events.emit({
        name: "agent.interrupted",
        data: {
          reason: event.data.reason,
          forced: event.data.force ?? false,
          author: event.data.author
        }
      });
    }
  }
  #processResourcesResponseEvent(event) {
    this.#resourcesResponses[event.data.requestId] = event.data.resources;
  }
  #processDecideEvent(generation, event) {
    const id = newId("decide");
    this.#decidePromises.push({
      id,
      event,
      promise: (async () => {
        const result = await this.#agent.models.llm.generateObject({
          messages: [
            {
              id: newId("message"),
              createdAt: Date.now(),
              lastUpdated: Date.now(),
              role: "system",
              content: `
                    # Instructions
                    You're a decision assistant helping another assistant itself directly discussing with
                    the user and interacting with the application. A new system information has been received
                    and your role is to decide whether the other assistant should react to this new information,
                    or just be passive. You'll be provided the conversation history including the new information,
                    as part of the last system messages.

                    ## Output
                    Once you've taken your decision, you'll output a 'shouldReact' boolean, indicating whether
                    that new information is worth reacting to and could help the conversation goal.

                    ## Recent conversation history
                    Here is the recent conversation history, including the new information.
                    Should the agent react to the most recent system messages?

                    ${event.data.messages.map((message) => {
                if (message.role === "user" || message.role === "system" || message.role === "agent")
                  return `${message.role}: ${message.content}`;
                else return "";
              }).join("\n")}
                    `
            }
          ],
          schema: z11.object({
            shouldContinue: z11.boolean()
          })
        });
        if (result.type === "content" && result.data.shouldContinue && generation.progress === "idle") {
          this.#plugin.events.emit({
            name: "agent.continue",
            data: event.data,
            urgent: true
          });
        }
        this.#decidePromises = this.#decidePromises.filter((decide) => decide.id !== id);
      })(),
      cancel: /* @__PURE__ */ __name(() => {
        this.#decidePromises = this.#decidePromises.filter((decide) => decide.id !== id);
        return event;
      }, "cancel")
    });
  }
  #processInsertEvent(generation, event) {
    if (event.data.interrupt === "abrupt")
      this.#processInterruptEvent({
        id: newId("event"),
        name: "agent.interrupt",
        data: {
          reason: "Interrupted by another operation.",
          author: "application",
          force: false
        },
        urgent: true,
        created: {
          at: Date.now(),
          by: {
            type: "handler",
            plugin: "generation",
            handler: "orchestrator",
            event: "unknown"
          }
        },
        edited: false,
        dropped: false,
        contextChanges: []
      });
    for (const decide of this.#decidePromises) {
      decide.cancel();
      generation.addInsertEvent({
        id: newId("event"),
        name: "agent.continue",
        data: decide.event.data,
        urgent: true,
        created: {
          at: Date.now(),
          by: {
            type: "handler",
            plugin: "generation",
            handler: "orchestrator",
            event: "unknown"
          }
        },
        edited: false,
        dropped: false,
        contextChanges: []
      });
    }
    generation.addInsertEvent(event);
    return event.data.interrupt === "smooth";
  }
  #isGenerationEvent(event) {
    return event.name === "agent.continue" || event.name === "agent.say" || event.name === "agent.decide" || event.name === "agent.resources-response" || event.name === "agent.interrupt" || event.name === "agent.speaking-end";
  }
  #isQueueBusy() {
    return this.#eventsQueue.some((event) => this.#isGenerationEvent(event));
  }
  async #consumeGenerations() {
    const limiter = throttledGenerationQueue(150);
    for await (const generation of this.#generationsQueue) {
      for await (const chunk of limiter(generation.queue)) {
        if (!this.#plugin.context.get().status.speaking && chunk.type === "content") {
          this.#plugin.events.emit({ name: "agent.speaking-start", urgent: true });
        }
        if (chunk.type === "content") {
          if (chunk.textChunk.length)
            this.#plugin.events.emit({
              name: "agent.text-chunk",
              data: { textChunk: chunk.textChunk }
            });
          if (this.#plugin.context.get().voiceEnabled && chunk.voiceChunk?.length)
            this.#plugin.events.emit({
              name: "agent.voice-chunk",
              data: { voiceChunk: chunk.voiceChunk }
            });
        } else if (chunk.type === "tool-requests") {
          if (chunk.requests.length)
            this.#plugin.events.emit({
              name: "agent.tool-requests",
              data: { requests: chunk.requests },
              urgent: true
            });
        } else if (chunk.type === "end") {
          this.#generations = this.#generations.filter((g) => g.id !== generation.id);
          if (this.#plugin.context.get().status.speaking && this.#generationsQueue.length() === 0) {
            this.#plugin.events.emit({ name: "agent.speaking-end", urgent: true });
          }
          break;
        }
      }
    }
  }
};
function throttledGenerationQueue(leadMs = 300, sampleRate = 16e3) {
  let anchorWallTime = Date.now();
  let totalAudioDurationMs = 0;
  const sleep = /* @__PURE__ */ __name((ms) => new Promise((r) => setTimeout(r, ms)), "sleep");
  return async function* (source) {
    for await (const chunk of source) {
      if (chunk.type === "content" && chunk.voiceChunk) {
        const chunkDurationMs = chunk.voiceChunk.length / sampleRate * 1e3;
        const wallElapsed = Date.now() - anchorWallTime;
        let lead = totalAudioDurationMs - wallElapsed;
        if (lead < -leadMs) {
          anchorWallTime += -lead - leadMs;
          lead = -leadMs;
        }
        if (lead > leadMs) {
          await sleep(lead - leadMs);
        }
        yield chunk;
        totalAudioDurationMs += chunkDurationMs;
      } else {
        yield chunk;
      }
    }
  };
}
__name(throttledGenerationQueue, "throttledGenerationQueue");

// plugins/defaults/generation/server/index.ts
var generationPlugin = definePlugin("generation").$config(generationPluginConfigDef).$events(generationPluginEventsDef).$context(generationPluginContextDef).addHandler({
  name: "maintain-status",
  mode: "block",
  onEvent: /* @__PURE__ */ __name(({ plugin, event }) => {
    if (!/^agent\.(thinking|speaking)/.test(event.name)) return;
    const status = dataOrThrow(computeStatus(plugin.context.get().status, event.name));
    plugin.context.set((ctx) => ({ ...ctx, status }));
  }, "onEvent")
}).addHandler({
  name: "maintain-messages",
  mode: "block",
  /**
   * @returns The id of the created or updated message (if any).
   */
  onEvent: /* @__PURE__ */ __name(({ event, plugin }) => {
    const messages = new MessageList(plugin.context.get().messages);
    let updatedMessageId;
    if (event.name === "messages.create") {
      updatedMessageId = dataOrThrow(messages.create(event.data.message));
    } else if (event.name === "messages.update") {
      updatedMessageId = dataOrThrow(
        messages.update(event.data.id, event.data.role, event.data.message)
      );
    } else if (event.name === "user.text-chunk") {
      const message = dataOrThrow(messages.findLastFromRoles(["user", "agent", "tool"]));
      if (message?.role === "user") {
        const content = `${message.content}${event.data.textChunk}`;
        updatedMessageId = dataOrThrow(messages.update(message.id, "user", { content }));
      } else
        updatedMessageId = dataOrThrow(
          messages.create({ role: "user", content: event.data.textChunk })
        );
    } else if (event.name === "user.interrupted") {
      const message = dataOrThrow(messages.findLastFromRoles(["user"]));
      if (!message) return;
      const content = `${message.content} [You interrupted the user, you might want to quickly apologize and mention that you're caring about what the user said]`;
      updatedMessageId = dataOrThrow(messages.update(message.id, "user", { content }));
    } else if (event.name === "agent.tool-requests") {
      const message = dataOrThrow(messages.findLastFromRoles(["user", "agent"]));
      if (message?.role === "agent") {
        const toolsRequests = [...message.toolsRequests, ...event.data.requests];
        updatedMessageId = dataOrThrow(
          messages.update(message.id, "agent", { toolsRequests })
        );
      } else {
        updatedMessageId = dataOrThrow(
          messages.create({ role: "agent", toolsRequests: event.data.requests })
        );
      }
    } else if (event.name === "agent.interrupted") {
      const message = dataOrThrow(messages.findLastFromRoles(["agent"]));
      if (!message)
        throw lifeError({
          code: "NotFound",
          message: "No agent message found. Should not happen."
        });
      if (!message.content.includes("[Interrupted")) {
        const content = `${message.content} [Interrupted by ${event.data.author}]`;
        updatedMessageId = dataOrThrow(messages.update(message.id, "agent", { content }));
      }
    } else if (event.name === "agent.text-chunk") {
      const message = dataOrThrow(messages.findLastFromRoles(["user", "agent", "tool"]));
      if (message?.role === "agent" && !message.content.includes("[Interrupted")) {
        const content = `${message.content}${event.data.textChunk}`;
        updatedMessageId = dataOrThrow(messages.update(message.id, "agent", { content }));
      }
    } else if (event.name === "agent.thinking-start") {
      updatedMessageId = dataOrThrow(messages.create({ role: "agent" }));
    }
    const messagesRaw = dataOrThrow(messages.getAll());
    plugin.context.set((ctx) => ({ ...ctx, messages: messagesRaw }));
    return updatedMessageId;
  }, "onEvent")
}).addHandler({
  name: "receive-user-audio",
  mode: "block",
  state: { unsubscribe: null, count: 9 },
  onEvent: /* @__PURE__ */ __name(({ event, plugin, state, agent }) =>