UNPKG

agent-swarm-kit

Version:

A TypeScript library for building orchestrated framework-agnostic multi-agent AI systems

927 lines (917 loc) 1.3 MB
import { scoped } from 'di-scoped'; import { createActivator } from 'di-kit'; import { trycatch, makeExtendable, singleshot, getErrorMessage, not, queued, memoize, retry, Subject, str, randomString, ToolRegistry, createAwaiter, sleep, errorData, isObject as isObject$1, cancelable, CANCELED_PROMISE_SYMBOL, SortedArray, execpool, ttl, compose, LimitedSet, Source, and, singlerun, schedule, rate, fetchApi } from 'functools-kit'; import xml2js from 'xml2js'; import fs, { access, mkdir } from 'fs/promises'; import path, { join } from 'path'; import crypto, { createHash } from 'crypto'; import os from 'os'; import { getMomentStamp, getTimeStamp } from 'get-moment-stamp'; /** * Scoped service class providing method call context information in the swarm system. * Stores and exposes an IMethodContext object (clientId, methodName, agentName, etc.) via dependency injection, scoped using di-scoped for method-specific instances. * Integrates with ClientAgent (e.g., EXECUTE_FN method context), PerfService (e.g., computeClientState with METHOD_NAME_COMPUTE_STATE), LoggerService (e.g., context logging in info), and DocService (e.g., documenting method-related entities). * Provides a lightweight, immutable context container for tracking method invocation metadata across the system. */ const MethodContextService = scoped(class { /** * Creates an instance of MethodContextService with the provided method context. * Stores the context immutably, making it available to dependent services like LoggerService and PerfService via DI. * @param {IMethodContext} context - The method context object containing clientId, methodName, agentName, swarmName, storageName, stateName, and policyName. */ constructor(context) { this.context = context; } }); const { init, inject, provide } = createActivator("agent-swarm"); const baseServices$1 = { busService: Symbol('busService'), docService: Symbol('docService'), perfService: Symbol('perfService'), aliveService: Symbol('aliveService'), loggerService: Symbol('loggerService'), }; const contextServices$1 = { methodContextService: Symbol('methodContextService'), payloadContextService: Symbol('payloadContextService'), executionContextService: Symbol('executionContextService'), schemaContextService: Symbol('schemaContextService'), }; const connectionServices$1 = { agentConnectionService: Symbol('agentConnectionService'), historyConnectionService: Symbol('historyConnectionService'), swarmConnectionService: Symbol('swarmConnectionService'), sessionConnectionService: Symbol('sessionConnectionService'), storageConnectionService: Symbol('storageConnectionService'), sharedStorageConnectionService: Symbol('sharedStorageConnectionService'), stateConnectionService: Symbol('stateConnectionService'), sharedStateConnectionService: Symbol('sharedStateConnectionService'), policyConnectionService: Symbol('policyConnectionService'), mcpConnectionService: Symbol('mcpConnectionService'), computeConnectionService: Symbol('computeConnectionService'), sharedComputeConnectionService: Symbol('sharedComputeConnectionService'), }; const schemaServices$1 = { completionSchemaService: Symbol('completionSchemaService'), agentSchemaService: Symbol('agentSchemaService'), swarmSchemaService: Symbol('swarmSchemaService'), toolSchemaService: Symbol('toolSchemaService'), embeddingSchemaService: Symbol('embeddingSchemaService'), storageSchemaService: Symbol('storageSchemaService'), stateSchemaService: Symbol('stateSchemaService'), memorySchemaService: Symbol('memorySchemaService'), policySchemaService: Symbol('policySchemaService'), wikiSchemaService: Symbol('wikiSchemaService'), mcpSchemaService: Symbol('mcpSchemaService'), computeSchemaService: Symbol('computeSchemaService'), pipelineSchemaService: Symbol('pipelineSchemaService'), navigationSchemaService: Symbol('navigationSchemaService'), }; const metaServices$1 = { agentMetaService: Symbol('agentMetaService'), swarmMetaService: Symbol('swarmMetaService'), }; const publicServices$1 = { agentPublicService: Symbol('agentPublicService'), historyPublicService: Symbol('historyPublicService'), sessionPublicService: Symbol('sessionPublicService'), swarmPublicService: Symbol('swarmPublicService'), storagePublicService: Symbol('storagePublicService'), sharedStoragePublicService: Symbol('sharedStoragePublicService'), statePublicService: Symbol('statePublicService'), sharedStatePublicService: Symbol('sharedStatePublicService'), policyPublicService: Symbol('policyPublicService'), mcpPublicService: Symbol('mcpPublicService'), computePublicService: Symbol('computePublicService'), sharedComputePublicService: Symbol('sharedComputePublicService'), }; const validationServices$1 = { agentValidationService: Symbol('agentValidationService'), toolValidationService: Symbol('toolValidationService'), sessionValidationService: Symbol('sessionValidationService'), swarmValidationService: Symbol('swarmValidationService'), completionValidationService: Symbol('completionValidationService'), embeddingValidationService: Symbol('embeddingValidationService'), storageValidationService: Symbol('storageValidationService'), policyValidationService: Symbol('policyValidationService'), navigationValidationService: Symbol('navigationValidationService'), wikiValidationService: Symbol('wikiValidationService'), mcpValidationService: Symbol('mcpValidationService'), computeValidationService: Symbol('computeValidationService'), stateValidationService: Symbol('stateValidationService'), pipelineValidationService: Symbol('pipelineValidationService'), executionValidationService: Symbol('executionValidationService'), }; const TYPES = { ...baseServices$1, ...contextServices$1, ...schemaServices$1, ...connectionServices$1, ...publicServices$1, ...validationServices$1, ...metaServices$1, }; /** * Scoped service class providing execution context information in the swarm system. * Stores and exposes an IExecutionContext object (clientId, executionId, processId) via dependency injection, scoped using di-scoped for execution-specific instances. * Integrates with ClientAgent (e.g., EXECUTE_FN execution context), PerfService (e.g., execution tracking in startExecution), BusService (e.g., execution events in commitExecutionBegin), and LoggerService (e.g., context logging in info). * Provides a lightweight, immutable context container for tracking execution metadata across the system. */ const ExecutionContextService = scoped(class { /** * Creates an instance of ExecutionContextService with the provided execution context. * Stores the context immutably, making it available to dependent services like LoggerService and BusService via DI. * @param {IExecutionContext} context - The execution context object containing clientId, executionId, and processId. */ constructor(context) { this.context = context; } }); /** * Validates that an output string is not empty or whitespace-only. * Trims the input and checks if the result is an empty string, returning an error message if so. * * @param {string} output - The output string to validate, typically from an agent or model response. * @returns {Promise<string | null>} A promise that resolves to: * - `"Empty output"` if the string is empty or contains only whitespace after trimming. * - `null` if the string has non-whitespace content, indicating it is valid. * * @example * // Basic usage with various inputs * console.log(await validateNoEmptyResult("Hello")); // null (valid) * console.log(await validateNoEmptyResult("")); // "Empty output" * console.log(await validateNoEmptyResult(" ")); // "Empty output" (whitespace only) * console.log(await validateNoEmptyResult(" Text ")); // null (valid after trim) * * @example * // Edge cases with falsy or special inputs * console.log(await validateNoEmptyResult("\n\t")); // "Empty output" (only newlines/tabs) * console.log(await validateNoEmptyResult("x")); // null (single character is valid) * * @remarks * This function: * - Uses `String.prototype.trim()` to remove leading and trailing whitespace from the input. * - Returns `"Empty output"` if the trimmed result is an empty string (`""`), treating whitespace-only strings as invalid. * - Returns `null` if the trimmed string has any non-whitespace characters, indicating a valid output. * - Operates asynchronously to align with the validation pipeline (e.g., `validateDefault`), though the check itself is synchronous. * Useful in the agent swarm system to ensure agent or model outputs contain meaningful content, preventing empty or trivial responses * from being processed further (e.g., as part of `ClientAgent.validate`). * * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/trim|String.trim} * for details on whitespace trimming. * @see {@link ./validateDefault|validateDefault} for its use in the broader validation chain. */ const validateNoEmptyResult = async (output) => { if (!output.trim()) { return "Empty output"; } return null; }; const toolParser = new xml2js.Parser(); /** * Validates that an output string contains no tool call entries or disallowed symbols. * Checks for specific symbols and XML tags defined in `GLOBAL_CONFIG`, returning an error message if any are found. * * @param {string} output - The output string to validate, typically from an agent or model response. * @returns {Promise<string | null>} A promise that resolves to: * - `"Tool call in text output"` if the string contains disallowed symbols (e.g., `{`, `}`) or XML tags (e.g., `<tool_call>`). * - `null` if no tool calls or disallowed symbols are detected, indicating the output is valid. * @throws {Error} Propagates unhandled errors from XML parsing if `trycatch` fails to catch them, though typically returns `null` on parse failure. * * @example * // Basic usage with valid and invalid outputs * console.log(await validateNoToolCall("Hello, world!")); // null (valid) * console.log(await validateNoToolCall("Use {tool} now")); // "Tool call in text output" (disallowed symbol) * console.log(await validateNoToolCall("<tool_call>run</tool_call>")); // "Tool call in text output" (disallowed tag) * * @example * // Edge cases with malformed XML or mixed content * console.log(await validateNoToolCall("<invalid>")); // null (parsing fails silently, returns default) * console.log(await validateNoToolCall("Text <tool>action</tool>")); // "Tool call in text output" (disallowed tag) * * @remarks * This function: * - Uses `trycatch` from `functools-kit` to wrap the validation logic, returning `null` if an error occurs (e.g., XML parsing failure). * - Checks for disallowed symbols (from `GLOBAL_CONFIG.CC_AGENT_DISALLOWED_SYMBOLS`, e.g., `{`, `}`) using `String.includes`. * - Parses the output as XML with `xml2js.Parser` and checks for disallowed tags (from `GLOBAL_CONFIG.CC_AGENT_DISALLOWED_TAGS`, * e.g., `tool_call`, `tool`) in the parsed result. * - Returns `"Tool call in text output"` as a generic error message for any detected tool call indicator, whether symbol- or tag-based. * - Handles malformed XML gracefully due to `trycatch`, avoiding crashes but potentially masking parsing issues. * Useful in the agent swarm system to filter out tool call artifacts from agent outputs (e.g., in `ClientAgent.validate`), ensuring clean text responses. * See the related GitHub issue for context on tool call handling challenges in similar systems. * * @see {@link https://github.com/ollama/ollama/issues/8287|GitHub Issue #8287} for background on tool call validation needs. * @see {@link ../config/params|GLOBAL_CONFIG} for disallowed symbols and tags configuration. * @see {@link https://www.npmjs.com/package/xml2js|xml2js} for XML parsing details. * @see {@link module:functools-kit.trycatch|trycatch} for error handling mechanics. */ const validateNoToolCall = trycatch(async (output) => { for (const symbol of GLOBAL_CONFIG.CC_AGENT_DISALLOWED_SYMBOLS) { if (output.includes(symbol)) { return "Tool call in text output"; } } const result = await toolParser.parseStringPromise(output); for (const tag of GLOBAL_CONFIG.CC_AGENT_DISALLOWED_TAGS) { if (result[tag]) { return "Tool call in text output"; } } return null; }, { defaultValue: null, }); /** * Validates an output string using a sequence of predefined validation functions. * Checks the string against multiple criteria (e.g., non-empty, no tool calls) and returns the first validation error encountered. * * @param {string} output - The output string to validate, typically from an agent or model response. * @returns {Promise<string | null>} A promise that resolves to: * - A string error message if any validation fails (e.g., "Output is empty" or "Contains tool call"). * - `null` if all validations pass, indicating the output is valid. * @throws {Error} Propagates any errors thrown by the underlying validation functions (`validateNoEmptyResult`, `validateNoToolCall`). * * @example * // Basic usage with valid and invalid outputs * console.log(await validateDefault("Hello, world!")); // null (valid) * console.log(await validateDefault("")); // "Output is empty" (assuming validateNoEmptyResult behavior) * console.log(await validateDefault("<tool_call>")); // "Contains tool call" (assuming validateNoToolCall behavior) * * @example * // Chained validation with multiple checks * const output = " "; // Only whitespace * const result = await validateDefault(output); * console.log(result); // "Output is empty" (assuming validateNoEmptyResult trims and checks) * * @remarks * This function: * - Sequentially applies validation functions (`validateNoEmptyResult`, `validateNoToolCall`) to the input `output`. * - Returns the first non-null result from a validation function, short-circuiting further checks. * - Relies on imported validators: * - `validateNoEmptyResult`: Likely checks for empty or whitespace-only strings. * - `validateNoToolCall`: Likely detects XML-like tool call tags (e.g., `<tool_call>`). * - Returns `null` only if all validators pass, indicating a fully valid output. * Useful in the agent swarm system as a default validation step for agent outputs, ensuring they meet basic quality criteria * before further processing or emission (e.g., in `ClientAgent.validate`). * * @see {@link ./validateNoEmptyResult} for the empty result validation logic. * @see {@link ./validateNoToolCall} for the tool call detection logic. */ const validateDefault = async (output) => { let validation = null; if ((validation = await validateNoEmptyResult(output))) { return validation; } if ((validation = await validateNoToolCall(output))) { return validation; } return null; }; /** * Removes XML tags and their contents from a string, cleaning up excess whitespace. * Strips all matched XML-like tags (e.g., `<tag>content</tag>`) and normalizes newlines, returning a trimmed result. * * @param {string} input - The input string containing potential XML tags to remove. Can be empty or falsy. * @returns {string} The cleaned string with XML tags and their contents removed, excess newlines collapsed, and leading/trailing whitespace trimmed. * Returns an empty string if the input is falsy. * * @example * // Basic usage with XML tags * console.log(removeXmlTags("<p>Hello</p>")); // "Hello" * console.log(removeXmlTags("<div><span>Text</span></div>")); // "Text" * console.log(removeXmlTags("No tags here")); // "No tags here" * * @example * // Handling multiline input and edge cases * console.log(removeXmlTags("<tag>\nLine1\nLine2\n</tag>")); // "Line1\nLine2" * console.log(removeXmlTags("<tag> <nested>Text</nested> </tag>")); // "Text" * console.log(removeXmlTags("")); // "" * console.log(removeXmlTags(null)); // "" * * @remarks * This function: * - Returns an empty string (`""`) for falsy inputs (e.g., `null`, `undefined`, empty string) to ensure safe handling. * - Uses a regular expression (`/<[^>]+>[\s\S]*?<\/[^>]+>/g`) to match and remove XML tags and their contents, including nested tags, * where `[\s\S]*?` ensures non-greedy matching across newlines. * - Collapses multiple consecutive newlines (`\n\s*\n`) into a single newline (`\n`) to clean up formatting. * - Trims leading and trailing whitespace from the final result for a polished output. * Useful in the agent swarm system for sanitizing model outputs, user inputs, or logs that may contain XML markup, * such as when processing tool call responses or cleaning structured text. * * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions|Regular Expressions} * for details on the regex patterns used. * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace|String.replace} * for string replacement mechanics. */ const removeXmlTags = (input) => { if (!input) { return ""; } return input .replace(/<[^>]+>[\s\S]*?<\/[^>]+>/g, "") .replace(/\n\s*\n/g, "\n") .trim(); }; const IS_WINDOWS = os.platform() === "win32"; /** * Atomically writes data to a file, ensuring the operation either fully completes or leaves the original file unchanged. * Uses a temporary file with a rename strategy on POSIX systems for atomicity, or direct writing with sync on Windows (or when POSIX rename is skipped). * * @param {string} file - The path to the target file to write. * @param {string | Buffer} data - The data to write, either as a string or Buffer. * @param {Options | BufferEncoding} [options={}] - Optional settings or a string encoding shorthand. * @param {BufferEncoding} [options.encoding="utf8"] - The encoding for the data (e.g., 'utf8', 'binary'). * @param {number} [options.mode=0o666] - The file mode (permissions) as an octal number. * @param {string} [options.tmpPrefix=".tmp-"] - The prefix for the temporary file name used during writing. * @returns {Promise<void>} A promise that resolves when the write completes successfully, or rejects with an error if the operation fails. * @throws {Error} Throws an error if the write, sync, or rename operation fails, after attempting cleanup of temporary files. * * @example * // Basic usage with default options * await writeFileAtomic("output.txt", "Hello, world!"); * // Writes "Hello, world!" to "output.txt" atomically * * @example * // Custom options and Buffer data * const buffer = Buffer.from("Binary data"); * await writeFileAtomic("data.bin", buffer, { encoding: "binary", mode: 0o644, tmpPrefix: "temp-" }); * // Writes binary data to "data.bin" with custom permissions and temp prefix * * @example * // Using encoding shorthand * await writeFileAtomic("log.txt", "Log entry", "utf16le"); * // Writes "Log entry" to "log.txt" in UTF-16LE encoding * * @remarks * This function ensures atomicity to prevent partial writes: * - On POSIX systems (non-Windows, unless `GLOBAL_CONFIG.CC_SKIP_POSIX_RENAME` is true): * - Writes data to a temporary file (e.g., `.tmp-<random>-filename`) in the same directory. * - Uses `crypto.randomBytes` to generate a unique temporary name, reducing collision risk. * - Syncs the data to disk and renames the temporary file to the target file atomically with `fs.rename`. * - Cleans up the temporary file on failure, swallowing cleanup errors to prioritize throwing the original error. * - On Windows (or when POSIX rename is skipped): * - Writes directly to the target file, syncing data to disk to minimize corruption risk (though not fully atomic). * - Closes the file handle on failure without additional cleanup. * - Accepts `options` as an object or a string (interpreted as `encoding`), defaulting to `{ encoding: "utf8", mode: 0o666, tmpPrefix: ".tmp-" }`. * Useful in the agent swarm system for safely writing configuration files, logs, or state data where partial writes could cause corruption. * * @see {@link https://nodejs.org/api/fs.html#fspromiseswritefilefile-data-options|fs.promises.writeFile} for file writing details. * @see {@link https://nodejs.org/api/crypto.html#cryptorandombytessize-callback|crypto.randomBytes} for temporary file naming. * @see {@link ../config/params|GLOBAL_CONFIG} for configuration impacting POSIX behavior. */ async function writeFileAtomic(file, data, options = {}) { if (typeof options === "string") { options = { encoding: options }; } else if (!options) { options = {}; } const { encoding = "utf8", mode = 0o666, tmpPrefix = ".tmp-" } = options; let fileHandle = null; if (IS_WINDOWS || GLOBAL_CONFIG.CC_SKIP_POSIX_RENAME) { try { // Create and write to temporary file fileHandle = await fs.open(file, "w", mode); // Write data to the temp file await fileHandle.writeFile(data, { encoding }); // Ensure data is flushed to disk await fileHandle.sync(); // Close the file before rename await fileHandle.close(); } catch (error) { // Clean up if something went wrong if (fileHandle) { await fileHandle.close().catch(() => { }); } throw error; // Re-throw the original error } return; } // Create a temporary filename in the same directory const dir = path.dirname(file); const filename = path.basename(file); const tmpFile = path.join(dir, `${tmpPrefix}${crypto.randomBytes(6).toString("hex")}-${filename}`); try { // Create and write to temporary file fileHandle = await fs.open(tmpFile, "w", mode); // Write data to the temp file await fileHandle.writeFile(data, { encoding }); // Ensure data is flushed to disk await fileHandle.sync(); // Close the file before rename await fileHandle.close(); fileHandle = null; // Atomically replace the target file with our temp file await fs.rename(tmpFile, file); } catch (error) { // Clean up if something went wrong if (fileHandle) { await fileHandle.close().catch(() => { }); } // Try to remove the temporary file try { await fs.unlink(tmpFile).catch(() => { }); } catch (_) { // Ignore errors during cleanup } throw error; } } var _a$4, _b$2, _c$1, _d$1, _e; /** @private Symbol for memoizing the wait-for-initialization operation in PersistBase */ const BASE_WAIT_FOR_INIT_SYMBOL = Symbol("wait-for-init"); /** @private Symbol for creating a new key in a persistent list */ const LIST_CREATE_KEY_SYMBOL = Symbol("create-key"); /** @private Symbol for getting the last key in a persistent list */ const LIST_GET_LAST_KEY_SYMBOL = Symbol("get-last-key"); /** @private Symbol for popping the last item from a persistent list */ const LIST_POP_SYMBOL = Symbol("pop"); // Logging method names for PersistBase /** @private Constant for logging the constructor in PersistBase */ const PERSIST_BASE_METHOD_NAME_CTOR = "PersistBase.CTOR"; /** @private Constant for logging the waitForInit method in PersistBase */ const PERSIST_BASE_METHOD_NAME_WAIT_FOR_INIT = "PersistBase.waitForInit"; /** @private Constant for logging the readValue method in PersistBase */ const PERSIST_BASE_METHOD_NAME_READ_VALUE = "PersistBase.readValue"; /** @private Constant for logging the writeValue method in PersistBase */ const PERSIST_BASE_METHOD_NAME_WRITE_VALUE = "PersistBase.writeValue"; /** @private Constant for logging the hasValue method in PersistBase */ const PERSIST_BASE_METHOD_NAME_HAS_VALUE = "PersistBase.hasValue"; /** @private Constant for logging the removeValue method in PersistBase */ const PERSIST_BASE_METHOD_NAME_REMOVE_VALUE = "PersistBase.removeValue"; /** @private Constant for logging the removeAll method in PersistBase */ const PERSIST_BASE_METHOD_NAME_REMOVE_ALL = "PersistBase.removeAll"; /** @private Constant for logging the values method in PersistBase */ const PERSIST_BASE_METHOD_NAME_VALUES = "PersistBase.values"; /** @private Constant for logging the keys method in PersistBase */ const PERSIST_BASE_METHOD_NAME_KEYS = "PersistBase.keys"; // Logging method names for PersistList /** @private Constant for logging the constructor in PersistList */ const PERSIST_LIST_METHOD_NAME_CTOR = "PersistList.CTOR"; /** @private Constant for logging the push method in PersistList */ const PERSIST_LIST_METHOD_NAME_PUSH = "PersistList.push"; /** @private Constant for logging the pop method in PersistList */ const PERSIST_LIST_METHOD_NAME_POP = "PersistList.pop"; // Logging method names for PersistSwarmUtils /** @private Constant for logging the usePersistActiveAgentAdapter method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_USE_PERSIST_ACTIVE_AGENT_ADAPTER = "PersistSwarmUtils.usePersistActiveAgentAdapter"; /** @private Constant for logging the usePersistNavigationStackAdapter method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_USE_PERSIST_NAVIGATION_STACK_ADAPTER = "PersistSwarmUtils.usePersistNavigationStackAdapter"; /** @private Constant for logging the getActiveAgent method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_GET_ACTIVE_AGENT = "PersistSwarmUtils.getActiveAgent"; /** @private Constant for logging the setActiveAgent method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_SET_ACTIVE_AGENT = "PersistSwarmUtils.setActiveAgent"; /** @private Constant for logging the getNavigationStack method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_GET_NAVIGATION_STACK = "PersistSwarmUtils.getNavigationStack"; /** @private Constant for logging the setNavigationStack method in PersistSwarmUtils */ const PERSIST_SWARM_UTILS_METHOD_NAME_SET_NAVIGATION_STACK = "PersistSwarmUtils.setNavigationStack"; // Logging method names for PersistStateUtils /** @private Constant for logging the usePersistStateAdapter method in PersistStateUtils */ const PERSIST_STATE_UTILS_METHOD_NAME_USE_PERSIST_STATE_ADAPTER = "PersistStateUtils.usePersistStateAdapter"; /** @private Constant for logging the setState method in PersistStateUtils */ const PERSIST_STATE_UTILS_METHOD_NAME_SET_STATE = "PersistStateUtils.setState"; /** @private Constant for logging the getState method in PersistStateUtils */ const PERSIST_STATE_UTILS_METHOD_NAME_GET_STATE = "PersistStateUtils.getState"; // Logging method names for PersistEmbeddingUtils /** @private Constant for logging the usePersistEmbeddingAdapter method in PersistEmbeddingUtils */ const PERSIST_EMBEDDING_UTILS_METHOD_NAME_USE_PERSIST_EMBEDDING_ADAPTER = "PersistEmbeddingUtils.usePersistEmbeddingAdapter"; /** @private Constant for logging the readEmbeddingCache method in PersistEmbeddingUtils */ const PERSIST_EMBEDDING_UTILS_METHOD_NAME_READ_EMBEDDING_CACHE = "PersistEmbeddingUtils.readEmbeddingCache"; /** @private Constant for logging the writeEmbeddingCache method in PersistEmbeddingUtils */ const PERSIST_EMBEDDING_UTILS_METHOD_NAME_WRITE_EMBEDDING_CACHE = "PersistEmbeddingUtils.writeEmbeddingCache"; // Logging method names for PersistMemoryUtils /** @private Constant for logging the usePersistMemoryAdapter method in PersistMemoryUtils */ const PERSIST_MEMORY_UTILS_METHOD_NAME_USE_PERSIST_MEMORY_ADAPTER = "PersistMemoryUtils.usePersistMemoryAdapter"; /** @private Constant for logging the setMemory method in PersistMemoryUtils */ const PERSIST_MEMORY_UTILS_METHOD_NAME_SET_MEMORY = "PersistMemoryUtils.setMemory"; /** @private Constant for logging the getMemory method in PersistMemoryUtils */ const PERSIST_MEMORY_UTILS_METHOD_NAME_GET_MEMORY = "PersistMemoryUtils.getMemory"; /** @private Constant for logging the dispose method in PersistMemoryUtils */ const PERSIST_MEMORY_UTILS_METHOD_NAME_DISPOSE = "PersistMemoryUtils.dispose"; // Logging method names for PersistAliveUtils /** @private Constant for logging the usePersistAliveAdapter method in PersistAliveUtils */ const PERSIST_ALIVE_UTILS_METHOD_NAME_USE_PERSIST_ALIVE_ADAPTER = "PersistAliveUtils.usePersistAliveAdapter"; /** @private Constant for logging the markOnline method in PersistAliveUtils */ const PERSIST_ALIVE_UTILS_METHOD_NAME_MARK_ONLINE = "PersistAliveUtils.markOnline"; /** @private Constant for logging the markOffline method in PersistAliveUtils */ const PERSIST_ALIVE_UTILS_METHOD_NAME_MARK_OFFLINE = "PersistAliveUtils.markOffline"; /** @private Constant for logging the getOnline method in PersistAliveUtils */ const PERSIST_ALIVE_UTILS_METHOD_NAME_GET_ONLINE = "PersistAliveUtils.getOnline"; // Logging method names for PersistStorageUtils /** @private Constant for logging the usePersistStorageAdapter method in PersistStorageUtils */ const PERSIST_STORAGE_UTILS_METHOD_NAME_USE_PERSIST_STORAGE_ADAPTER = "PersistStorageUtils.usePersistStorageAdapter"; /** @private Constant for logging the getData method in PersistStorageUtils */ const PERSIST_STORAGE_UTILS_METHOD_NAME_GET_DATA = "PersistStorageUtils.getData"; /** @private Constant for logging the setData method in PersistStorageUtils */ const PERSIST_STORAGE_UTILS_METHOD_NAME_SET_DATA = "PersistStorageUtils.setData"; const PERSIST_POLICY_UTILS_METHOD_NAME_USE_PERSIST_POLICY_ADAPTER = "PersistPolicyUtils.usePersistPolicyAdapter"; const PERSIST_POLICY_UTILS_METHOD_NAME_GET_BANNED_CLIENTS = "PersistPolicyUtils.getBannedClients"; const PERSIST_POLICY_UTILS_METHOD_NAME_SET_BANNED_CLIENTS = "PersistPolicyUtils.setBannedClients"; // Logging method names for private functions /** @private Constant for logging the waitForInitFn function */ const BASE_WAIT_FOR_INIT_FN_METHOD_NAME = "PersistBase.waitForInitFn"; /** @private Constant for logging the createKeyFn function */ const LIST_CREATE_KEY_FN_METHOD_NAME = "PersistList.createKeyFn"; /** @private Constant for logging the popFn function */ const LIST_POP_FN_METHOD_NAME = "PersistList.popFn"; /** @private Constant for logging the getLastKeyFn function */ const LIST_GET_LAST_KEY_FN_METHOD_NAME = "PersistList.getLastKeyFn"; /** @private Count of retry attempts for unlink in waitForInit */ const BASE_UNLINK_RETRY_COUNT = 5; /** @private Delay for retry attempts for unlink in waitForInit (in milliseconds) */ const BASE_UNLINK_RETRY_DELAY = 1000; /** * Attempts to remove a file if invalid JSON is detected during initialization. * Retries the operation multiple times with delays to handle transient errors, ensuring robust setup. * @private * @param {string} filePath - The path to the file to remove (e.g., `./logs/data/alive/<clientId>.json`). * @returns {Promise<boolean>} A promise resolving to `true` if the file is removed, `false` if removal fails after retries. */ const BASE_WAIT_FOR_INIT_UNLINK_FN = async (filePath) => trycatch(retry(async () => { try { await fs.unlink(filePath); return true; } catch (error) { console.error(`agent-swarm PersistBase unlink failed for filePath=${filePath} error=${getErrorMessage(error)}`); throw error; } }, BASE_UNLINK_RETRY_COUNT, BASE_UNLINK_RETRY_DELAY), { defaultValue: false, }); /** * Initializes the storage directory and validates existing entities, removing invalid ones. * Ensures the persistence layer is robust by cleaning up corrupted files during setup (e.g., for swarm or session data). * @private * @param {TPersistBase} self - The `PersistBase` instance being initialized, managing entities like alive status or agent states. * @returns {Promise<void>} A promise that resolves when initialization is complete. * @throws {Error} If directory creation or file validation fails (e.g., permissions or I/O errors). */ const BASE_WAIT_FOR_INIT_FN = async (self) => { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(BASE_WAIT_FOR_INIT_FN_METHOD_NAME, { entityName: self.entityName, directory: self._directory, }); await fs.mkdir(self._directory, { recursive: true }); for await (const key of self.keys()) { try { await self.readValue(key); } catch { const filePath = self._getFilePath(key); console.error(`agent-swarm PersistBase found invalid document for filePath=${filePath} entityName=${self.entityName}`); if (await not(BASE_WAIT_FOR_INIT_UNLINK_FN(filePath))) { console.error(`agent-swarm PersistBase failed to remove invalid document for filePath=${filePath} entityName=${self.entityName}`); } } } }; /** * Generates a new unique key for a list item by incrementing the last used key. * Initializes the last count by scanning existing keys if not already set, used in `PersistList` for ordered storage. * @private * @param {TPersistList} self - The `PersistList` instance generating the key, managing entities like message logs. * @returns {Promise<string>} A promise resolving to the new key as a string (e.g., "1", "2"). * @throws {Error} If key generation fails due to underlying storage issues (e.g., directory access). */ const LIST_CREATE_KEY_FN = async (self) => { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(LIST_CREATE_KEY_FN_METHOD_NAME, { entityName: self.entityName, lastCount: self._lastCount, }); if (self._lastCount === null) { for await (const key of self.keys()) { const numericKey = Number(key); if (!isNaN(numericKey)) { self._lastCount = Math.max(numericKey, self._lastCount || 0); } } } if (self._lastCount === null) { self._lastCount = 0; } return String((self._lastCount += 1)); }; /** * Removes and returns the last item from the persistent list. * Uses the last key to fetch and delete the item atomically, ensuring consistency in list operations. * @private * @param {TPersistList} self - The `PersistList` instance performing the pop operation, managing entities like event logs. * @returns {Promise<any | null>} A promise resolving to the removed item or `null` if the list is empty. * @throws {Error} If reading or removing the item fails (e.g., file not found or permissions). */ const LIST_POP_FN = async (self) => { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(LIST_POP_FN_METHOD_NAME, { entityName: self.entityName, }); const lastKey = await self[LIST_GET_LAST_KEY_SYMBOL](); if (lastKey === null) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(LIST_POP_FN_METHOD_NAME, { entityName: self.entityName, result: "No last key found, returning null", }); return null; } const value = await self.readValue(lastKey); await self.removeValue(lastKey); GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(LIST_POP_FN_METHOD_NAME, { entityName: self.entityName, lastKey, value, }); return value; }; /** * Retrieves the key of the last item in the persistent list by scanning all keys. * Determines the highest numeric key value for ordered list management (e.g., dequeuing events). * @private * @param {TPersistList} self - The `PersistList` instance retrieving the key, managing entities like history records. * @returns {Promise<string | null>} A promise resolving to the last key as a string (e.g., "5") or `null` if the list is empty. * @throws {Error} If key retrieval fails due to underlying storage issues (e.g., directory access). */ const LIST_GET_LAST_KEY_FN = async (self) => { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(LIST_GET_LAST_KEY_FN_METHOD_NAME, { entityName: self.entityName, }); let lastKey = 0; for await (const key of self.keys()) { const numericKey = Number(key); if (!isNaN(numericKey)) { lastKey = Math.max(numericKey, lastKey); } } if (lastKey === 0) { return null; } return String(lastKey); }; /** * Base class for persistent storage of entities in the swarm system, using the file system. * Provides foundational methods for reading, writing, and managing entities as JSON files, supporting swarm utilities like `PersistAliveUtils`. * @template EntityName - The type of entity name (e.g., `SwarmName`, `SessionId`), defaults to `string`, used as a subdirectory. * @implements {IPersistBase} */ const PersistBase = makeExtendable(class { /** * Creates a new `PersistBase` instance for managing persistent storage of entities. * Sets up the storage directory based on the entity name (e.g., `SwarmName` for swarm-specific data) and base directory. * @param {EntityName} entityName - The name of the entity type (e.g., `SwarmName` for swarm data, `SessionId` for memory), used as a subdirectory. * @param {string} [baseDir=join(process.cwd(), "logs/data")] - The base directory for storing entity files. */ constructor(entityName, baseDir = join(process.cwd(), "logs/data")) { this.entityName = entityName; this.baseDir = baseDir; /** * Memoized initialization function ensuring it runs only once per instance. * Uses `singleshot` to prevent redundant initialization calls, critical for swarm setup efficiency. * @private * @returns {Promise<void>} A promise that resolves when initialization is complete. */ this[_a$4] = singleshot(async () => await BASE_WAIT_FOR_INIT_FN(this)); GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_CTOR, { entityName: this.entityName, baseDir, }); this._directory = join(this.baseDir, this.entityName); } /** * Computes the full file path for an entity based on its ID. * @private * @param {EntityId} entityId - The identifier of the entity (string or number), unique within the entity type’s storage. * @returns {string} The full file path (e.g., `./logs/data/alive/<entityId>.json`). */ _getFilePath(entityId) { return join(this.baseDir, this.entityName, `${entityId}.json`); } /** * Initializes the storage directory, creating it if it doesn’t exist, and validates existing entities. * Removes invalid JSON files during initialization to ensure data integrity (e.g., for `SwarmName`-based alive status). * @param {boolean} initial - Indicates if this is the initial setup; reserved for future caching or optimization logic. * @returns {Promise<void>} A promise that resolves when initialization is complete. * @throws {Error} If directory creation or entity validation fails (e.g., permissions or I/O errors). */ async waitForInit(initial) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_WAIT_FOR_INIT, { entityName: this.entityName, initial, }); await this[BASE_WAIT_FOR_INIT_SYMBOL](); } /** * Retrieves the number of entities stored in the directory. * Counts only files with a `.json` extension, useful for monitoring storage usage (e.g., active sessions). * @returns {Promise<number>} A promise resolving to the count of stored entities. * @throws {Error} If reading the directory fails (e.g., permissions or directory not found). */ async getCount() { const files = await fs.readdir(this._directory); const { length } = files.filter((file) => file.endsWith(".json")); return length; } /** * Reads an entity from storage by its ID, parsing it from a JSON file. * Core method for retrieving persisted data (e.g., alive status for a `SessionId` in a `SwarmName` context). * @template T - The specific type of the entity (e.g., `IPersistAliveData`), defaults to `IEntity`. * @param {EntityId} entityId - The identifier of the entity to read (string or number), unique within its storage context. * @returns {Promise<T>} A promise resolving to the parsed entity data. * @throws {Error} If the file is not found (`ENOENT`) or parsing fails (e.g., invalid JSON). */ async readValue(entityId) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_READ_VALUE, { entityName: this.entityName, entityId, }); try { const filePath = this._getFilePath(entityId); const fileContent = await fs.readFile(filePath, "utf-8"); return JSON.parse(fileContent); } catch (error) { if (error?.code === "ENOENT") { throw new Error(`Entity ${this.entityName}:${entityId} not found`); } throw new Error(`Failed to read entity ${this.entityName}:${entityId}: ${getErrorMessage(error)}`); } } /** * Checks if an entity exists in storage by its ID. * Efficiently verifies presence without reading the full entity (e.g., checking if a `SessionId` has memory). * @param {EntityId} entityId - The identifier of the entity to check (string or number), unique within its storage context. * @returns {Promise<boolean>} A promise resolving to `true` if the entity exists, `false` otherwise. * @throws {Error} If checking existence fails for reasons other than the file not existing. */ async hasValue(entityId) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_HAS_VALUE, { entityName: this.entityName, entityId, }); try { const filePath = this._getFilePath(entityId); await fs.access(filePath); return true; } catch (error) { if (error?.code === "ENOENT") { return false; } throw new Error(`Failed to check existence of entity ${this.entityName}:${entityId}: ${getErrorMessage(error)}`); } } /** * Writes an entity to storage with the specified ID, serializing it to JSON. * Uses atomic file writing via `writeFileAtomic` to ensure data integrity (e.g., persisting `AgentName` for a `SwarmName`). * @template T - The specific type of the entity (e.g., `IPersistActiveAgentData`), defaults to `IEntity`. * @param {EntityId} entityId - The identifier for the entity (string or number), unique within its storage context. * @param {T} entity - The entity data to persist (e.g., `{ agentName: "agent1" }`). * @returns {Promise<void>} A promise that resolves when the write operation is complete. * @throws {Error} If writing to the file system fails (e.g., permissions or disk space). */ async writeValue(entityId, entity) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_WRITE_VALUE, { entityName: this.entityName, entityId, }); try { const filePath = this._getFilePath(entityId); const serializedData = JSON.stringify(entity); await writeFileAtomic(filePath, serializedData, "utf-8"); } catch (error) { throw new Error(`Failed to write entity ${this.entityName}:${entityId}: ${getErrorMessage(error)}`); } } /** * Removes an entity from storage by its ID. * Deletes the corresponding JSON file, used for cleanup (e.g., removing a `SessionId`’s memory). * @param {EntityId} entityId - The identifier of the entity to remove (string or number), unique within its storage context. * @returns {Promise<void>} A promise that resolves when the entity is deleted. * @throws {Error} If the entity is not found (`ENOENT`) or deletion fails (e.g., permissions). */ async removeValue(entityId) { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_REMOVE_VALUE, { entityName: this.entityName, entityId, }); try { const filePath = this._getFilePath(entityId); await fs.unlink(filePath); } catch (error) { if (error?.code === "ENOENT") { throw new Error(`Entity ${this.entityName}:${entityId} not found for deletion`); } throw new Error(`Failed to remove entity ${this.entityName}:${entityId}: ${getErrorMessage(error)}`); } } /** * Removes all entities from storage under this entity name. * Deletes all `.json` files in the directory, useful for resetting persistence (e.g., clearing a `SwarmName`’s data). * @returns {Promise<void>} A promise that resolves when all entities are removed. * @throws {Error} If reading the directory or deleting files fails (e.g., permissions). */ async removeAll() { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_REMOVE_ALL, { entityName: this.entityName, }); try { const files = await fs.readdir(this._directory); const entityFiles = files.filter((file) => file.endsWith(".json")); for (const file of entityFiles) { await fs.unlink(join(this._directory, file)); } } catch (error) { throw new Error(`Failed to remove values for ${this.entityName}: ${getErrorMessage(error)}`); } } /** * Iterates over all entities in storage, sorted numerically by ID. * Yields entities in ascending order, useful for batch processing (e.g., listing all `SessionId`s in a `SwarmName`). * @template T - The specific type of the entities (e.g., `IPersistAliveData`), defaults to `IEntity`. * @returns {AsyncGenerator<T>} An async generator yielding each entity in sorted order. * @throws {Error} If reading the directory or entity files fails (e.g., permissions or invalid JSON). */ async *values() { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_VALUES, { entityName: this.entityName, }); try { const files = await fs.readdir(this._directory); const entityIds = files .filter((file) => file.endsWith(".json")) .map((file) => file.slice(0, -5)) .sort((a, b) => a.localeCompare(b, undefined, { numeric: true, sensitivity: "base", })); for (const entityId of entityIds) { const entity = await this.readValue(entityId); yield entity; } } catch (error) { throw new Error(`Failed to read values for ${this.entityName}: ${getErrorMessage(error)}`); } } /** * Iterates over all entity IDs in storage, sorted numerically. * Yields IDs in ascending order, useful for key enumeration (e.g., listing `SessionId`s in a `SwarmName`). * @returns {AsyncGenerator<EntityId>} An async generator yielding each entity ID as a string or number. * @throws {Error} If reading the directory fails (e.g., permissions or directory not found). */ async *keys() { GLOBAL_CONFIG.CC_LOGGER_ENABLE_DEBUG && swarm$1.loggerService.debug(PERSIST_BASE_METHOD_NAME_KEYS, { entityName: this.entityName, }); try { const files = await fs.readdir(this._directory); const entityIds = files .filter((file) => file.endsWith(".json")) .map((file) => file.slice(0, -5)) .sort((a, b) => a.localeCompare(b, undefined, { numeric: true, sensitivity: "base", })); for (const entityId of entityIds) { yield entityId; } } catch (error) { throw new Error(`Failed to read keys for ${this.entityName}: ${getErrorMessage(error)}`); } } /** * Implements the async iterator protocol for iterating over entities. * Delegates to the `values` method for iteration, enabling `for await` loops over entities. * @returns {AsyncIterableIterator<any>} An async iterator yielding entities. */ async *[(_a$4 = BASE_WAIT_FOR_INIT_SYMBOL, Symbol.asyncIterator)]() { for await (const entity of this.values()) { yield entity; } } /** * Filters entities based on a predicate function, yielding only matching entities. * Useful for selective retrieval (e.g., finding online `SessionId`s in a `SwarmName`). * @template T - The specific type of the entities (e.g., `IPersistAliveData`), defaults to `IEntity`. * @param {(value: T) => boolean} predicate - A function to test each entity, returning `true` to include it. * @returns {AsyncGenerator<T>} An async generator yielding filtered entities in sorted order. * @throws {Error} If re