@ydbjs/topic
Version:
YDB Topics client for publish-subscribe messaging. Provides at-least-once delivery, exactly-once publishing, FIFO guarantees, and scalable message processing for unstructured data.
385 lines • 14 kB
TypeScript
/**
* @fileoverview YDB Topic Writer State Machine
*
* Implements a high-performance state machine for writing messages to YDB Topics.
* Handles connection management, batching, flow control, error recovery, shutdown, and memory management.
*
* Key Features:
* - Sliding window message buffer for efficient memory usage
* - Automatic batching with size and inflight constraints
* - Flow control via maxBufferBytes and maxInflightCount
* - Configurable garbage collection thresholds
* - Exponential backoff for error recovery
* - Resource cleanup on close/destroy
*
* Memory Management:
* Uses a single messages array with sliding window approach:
* [garbage...][inflight...][buffer...]
* ↑ ↑ ↑
* 0 inflightStart bufferStart
*
* Messages flow: buffer → inflight → garbage → compacted
*/
import type { Driver } from '@ydbjs/core';
import { type WriterStreamReceiveEvent } from './stream.js';
import type { TopicWriterOptions, WriterContext, WriterEmitted, WriterEvents, WriterInput } from './types.js';
export declare const MAX_BATCH_SIZE: bigint;
export declare const MAX_PAYLOAD_SIZE: bigint;
export declare const DEFAULT_GARBAGE_COUNT_THRESHOLD = 1000;
export declare const DEFAULT_GARBAGE_SIZE_THRESHOLD: bigint;
/**
* State machine for TopicWriter lifecycle management.
*
* Handles the complete lifecycle of a YDB Topic writer including:
* - Connection establishment and session initialization
* - Message batching and flow control
* - Graceful shutdown with pending message flush
* - Error handling with exponential backoff retry
* - Resource cleanup and memory management
*
* State Transitions:
* - connecting: Establish gRPC stream connection
* - connected: Initialize write session with YDB
* - ready: Idle state, waiting for messages or flush triggers
* - writing: Actively sending message batches
* - flushing: Manual flush, waiting for all acks
* - closing: Graceful shutdown, flushing pending messages
* - errored: Error state with retry logic
* - closed: Final state with all resources released
*/
export declare const WriterMachine: import("xstate").StateMachine<WriterContext, WriterEvents, {
[x: string]: import("xstate").ActorRefFromLogic<import("xstate").CallbackActorLogic<WriterStreamReceiveEvent, import("./stream.js").WriterStreamInput, import("./stream.js").WriterStreamEmittedEvent>> | undefined;
}, {
src: "WriterStream";
logic: import("xstate").CallbackActorLogic<WriterStreamReceiveEvent, import("./stream.js").WriterStreamInput, import("./stream.js").WriterStreamEmittedEvent>;
id: string | undefined;
}, import("xstate").Values<{
log: {
type: "log";
params: {
message: string;
};
};
createConnection: {
type: "createConnection";
params: import("xstate").NonReducibleUnknown;
};
closeConnection: {
type: "closeConnection";
params: import("xstate").NonReducibleUnknown;
};
createWriteSession: {
type: "createWriteSession";
params: import("xstate").NonReducibleUnknown;
};
updateWriteSession: {
type: "updateWriteSession";
params: import("xstate").NonReducibleUnknown;
};
sendMessages: {
type: "sendMessages";
params: import("xstate").NonReducibleUnknown;
};
acknowledgeMessages: {
type: "acknowledgeMessages";
params: import("xstate").NonReducibleUnknown;
};
enqueueMessage: {
type: "enqueueMessage";
params: import("xstate").NonReducibleUnknown;
};
resetAttempts: {
type: "resetAttempts";
params: import("xstate").NonReducibleUnknown;
};
incrementAttempts: {
type: "incrementAttempts";
params: import("xstate").NonReducibleUnknown;
};
recordError: {
type: "recordError";
params: import("xstate").NonReducibleUnknown;
};
reportError: {
type: "reportError";
params: import("xstate").NonReducibleUnknown;
};
releaseResources: {
type: "releaseResources";
params: import("xstate").NonReducibleUnknown;
};
}>, import("xstate").Values<{
allMessagesSent: {
type: "allMessagesSent";
params: unknown;
};
bufferFullAndCanSend: {
type: "bufferFullAndCanSend";
params: unknown;
};
hasMessagesAndCanSend: {
type: "hasMessagesAndCanSend";
params: unknown;
};
retryableError: {
type: "retryableError";
params: unknown;
};
nonRetryableError: {
type: "nonRetryableError";
params: unknown;
};
shouldReclaimMemory: {
type: "shouldReclaimMemory";
params: unknown;
};
}>, "retryDelay" | "flushInterval" | "gracefulShutdownTimeout", "ready" | "connecting" | "connected" | "errored" | "flushing" | "closing" | "idle" | "closed", string, WriterInput, import("xstate").NonReducibleUnknown, WriterEmitted, import("xstate").MetaObject, {
readonly id: "WriterMachine";
readonly initial: "idle";
readonly context: ({ input }: {
input: {
driver: Driver;
options: TopicWriterOptions;
};
}) => {
tx?: import("../tx.ts").TX;
messages: never[];
bufferStart: number;
bufferLength: number;
inflightStart: number;
inflightLength: number;
bufferSize: bigint;
inflightSize: bigint;
garbageSize: bigint;
messageGroupId?: string;
partitionId?: bigint;
dbg: import("@ydbjs/debug").YDBDebugLogger;
driver: Driver;
options: TopicWriterOptions;
attempts: number;
producerId: string;
};
readonly on: {
readonly 'writer.close': {
readonly target: ".closing";
readonly actions: readonly [{
type: "log";
params: {
message: string;
};
}];
};
readonly 'writer.destroy': {
readonly target: ".closed";
readonly actions: readonly [{
type: "log";
params: {
message: string;
};
}];
};
readonly 'writer.stream.error': {
readonly target: ".errored";
readonly actions: readonly ["recordError", {
type: "log";
params: {
message: string;
};
}];
};
};
readonly states: {
readonly idle: {
readonly always: {
readonly target: "connecting";
readonly actions: readonly [{
type: "log";
params: {
message: string;
};
}];
};
};
/**
* Connecting state: Establishes connection to the topic stream.
* - Buffers incoming messages while connecting.
* - Transitions to `connected` once the stream is ready.
*/
readonly connecting: {
readonly entry: readonly ["createConnection"];
readonly on: {
readonly 'writer.write': {
readonly actions: readonly ["enqueueMessage"];
};
readonly 'writer.stream.start': {
readonly target: "connected";
};
};
};
/**
* Connected state: Initializes the write session.
* - Buffers incoming messages.
* - Transitions to `ready` once the session is established.
*/
readonly connected: {
readonly entry: readonly ["createWriteSession", {
type: "log";
params: {
message: string;
};
}];
readonly on: {
readonly 'writer.write': {
readonly actions: readonly ["enqueueMessage"];
};
readonly 'writer.stream.close': {
readonly target: "connecting";
};
readonly 'writer.stream.response.init': {
readonly target: "ready";
readonly actions: readonly ["resetAttempts", "updateWriteSession", {
type: "log";
params: {
message: string;
};
}];
};
};
};
/**
* Ready state: Idle, waiting for buffer to fill or flush interval.
* - Automatically transitions to `writing` when buffer is full and can send messages.
* - Handles manual flush requests.
*/
readonly ready: {
readonly always: {
readonly guard: "bufferFullAndCanSend";
readonly actions: readonly ["sendMessages"];
};
readonly after: {
readonly flushInterval: {
readonly guard: "hasMessagesAndCanSend";
readonly actions: readonly ["sendMessages"];
};
};
readonly on: {
readonly 'writer.write': {
readonly actions: readonly ["enqueueMessage"];
};
readonly 'writer.flush': {
readonly target: "flushing";
};
readonly 'writer.stream.close': {
readonly target: "connecting";
};
readonly 'writer.stream.response.write': {
readonly actions: readonly ["acknowledgeMessages"];
};
};
};
/**
* Flushing state: Sends all buffered messages and waits for acknowledgments.
* - Blocks new writes during flush to avoid infinite loops.
* - Exits only when all messages are sent and acknowledged.
*/
readonly flushing: {
readonly always: readonly [{
readonly guard: "allMessagesSent";
readonly target: "ready";
}, {
readonly guard: "hasMessagesAndCanSend";
readonly target: "flushing";
readonly reenter: true;
readonly actions: readonly ["sendMessages"];
}];
readonly after: {
readonly flushInterval: {
readonly target: "flushing";
readonly reenter: true;
};
};
readonly on: {
readonly 'writer.stream.response.write': {
readonly actions: readonly ["acknowledgeMessages"];
};
};
};
/**
* Closing state: Performs graceful shutdown by flushing all messages before closing.
* - Ensures all messages are sent and acknowledged before closing.
* - Includes timeout mechanism to force close if graceful shutdown takes too long.
*/
readonly closing: {
readonly always: readonly [{
readonly target: "closed";
readonly guard: "allMessagesSent";
readonly actions: readonly ["closeConnection"];
}, {
readonly guard: "hasMessagesAndCanSend";
readonly target: "closing";
readonly reenter: true;
readonly actions: readonly ["sendMessages"];
}];
readonly after: {
readonly flushInterval: {
readonly guard: "hasMessagesAndCanSend";
readonly target: "closing";
readonly reenter: true;
};
readonly gracefulShutdownTimeout: {
readonly target: "closed";
readonly actions: readonly ["closeConnection"];
};
};
readonly on: {
readonly 'writer.stream.response.write': {
readonly target: "closing";
readonly reenter: true;
readonly actions: readonly ["acknowledgeMessages"];
};
};
};
/**
* Errored state: Handles errors and decides whether to retry or close.
* - Closes the connection immediately upon entering this state.
* - Transitions to `closed` for non-retryable errors.
* - Attempts reconnection for retryable errors after a delay.
* - Buffers incoming messages even while in error state.
*/
readonly errored: {
readonly entry: readonly ["closeConnection"];
readonly always: readonly [{
readonly guard: "nonRetryableError";
readonly target: "closed";
readonly actions: readonly ["reportError"];
}];
readonly after: {
readonly retryDelay: {
readonly guard: "retryableError";
readonly target: "connecting";
readonly actions: readonly ["resetAttempts"];
};
};
readonly on: {
readonly 'writer.write': {
readonly actions: readonly ["enqueueMessage"];
};
};
};
/**
* Closed state: Final state where all resources are released.
* - Ensures the connection is closed and resources are cleaned up.
* - No further transitions occur from this state.
*/
readonly closed: {
readonly type: "final";
readonly entry: readonly ["closeConnection", "releaseResources", {
type: "log";
params: {
message: string;
};
}];
};
};
}>;
//# sourceMappingURL=machine.d.ts.map