inngest/components/execution/otel/processor.cjs.map

Official SDK for Inngest.com. Inngest is the reliability layer for modern applications. Inngest combines durable execution, events, and queues into a zero-infra platform with built-in observability.

{"version":3,"file":"processor.cjs","names":["debugPrefix","_resourceAttributes: Resource | undefined","#traceParents","appId: string | undefined","functionId: string | undefined","traceRef: string | undefined","TraceStateKey","#checkpointingRoots","#activeStepContext","osDetector","envDetector","hostDetector","processDetector","serviceInstanceIdDetector","#batcher","getAsyncCtx","BatchSpanProcessor","OTLPTraceExporter","#spanCleanup","#spansToExport","deterministicSpanID","Attribute"],"sources":["../../../../src/components/execution/otel/processor.ts"],"sourcesContent":["import type { Span } from \"@opentelemetry/api\";\nimport { OTLPTraceExporter } from \"@opentelemetry/exporter-trace-otlp-http\";\nimport {\n  detectResources,\n  envDetector,\n  hostDetector,\n  osDetector,\n  processDetector,\n  type Resource,\n  serviceInstanceIdDetector,\n} from \"@opentelemetry/resources\";\nimport {\n  BatchSpanProcessor,\n  type ReadableSpan,\n  type SpanProcessor,\n} from \"@opentelemetry/sdk-trace-base\";\nimport Debug from \"debug\";\nimport { deterministicSpanID } from \"../../../helpers/deterministicId.ts\";\nimport type { Inngest } from \"../../Inngest.ts\";\nimport { getAsyncCtx } from \"../als.ts\";\nimport { clientProcessorMap } from \"./access.ts\";\nimport { Attribute, debugPrefix, TraceStateKey } from \"./consts.ts\";\n\nconst processorDevDebug = Debug(`${debugPrefix}:InngestSpanProcessor`);\n\n/**\n * A set of resource attributes that are used to identify the Inngest app and\n *  the function that is being executed. This is used to store the resource\n *  attributes for the spans that are exported to the Inngest endpoint, and cache\n *  them for later use.\n */\nlet _resourceAttributes: Resource | undefined;\n\n/**\n * A set of information about an execution that's used to set attributes on\n * userland spans sent to Inngest for proper indexing.\n */\nexport type ParentState = {\n  traceparent: string;\n  runId: string;\n  appId: string | undefined;\n  functionId: string | undefined;\n  traceRef: string | undefined;\n  rootSpanId: string;\n};\n\n/**\n * An OTel span processor that is used to export spans to the Inngest endpoint.\n * This is used to track spans that are created within an Inngest run and export\n * them to the Inngest endpoint for tracing.\n *\n * It's careful to only pick relevant spans to export and will not send any\n * irrelevant spans to the Inngest endpoint.\n *\n * THIS IS THE INTERNAL IMPLEMENTATION OF THE SPAN PROCESSOR AND SHOULD NOT BE\n * USED BY USERS DIRECTLY. USE THE {@link PublicInngestSpanProcessor} CLASS\n * INSTEAD.\n */\nexport class InngestSpanProcessor implements SpanProcessor {\n  /**\n   * An OTel span processor that is used to export spans to the Inngest endpoint.\n   * This is used to track spans that are created within an Inngest run and export\n   * them to the Inngest endpoint for tracing.\n   *\n   * It's careful to only pick relevant spans to export and will not send any\n   * irrelevant spans to the Inngest endpoint.\n   */\n  constructor(\n    /**\n     * The app that this span processor is associated with. This is used to\n     * determine the Inngest endpoint to export spans to.\n     *\n     * It is optional here as this is the private constructor and only used\n     * internally; we set `app` elsewhere as when we create the processor (as\n     * early as possible when the process starts) we don't necessarily have the\n     * app available yet.\n     *\n     * So, internally we can delay setting ths until later.\n     */\n    app?: Inngest.Like,\n  ) {\n    if (app) {\n      clientProcessorMap.set(app as Inngest.Any, this);\n    }\n  }\n\n  /**\n   * A `BatchSpanProcessor` that is used to export spans to the Inngest\n   * endpoint. This is created lazily to avoid creating it until the Inngest App\n   * has been initialized and has had a chance to receive environment variables,\n   * which may be from an incoming request.\n   */\n  #batcher: Promise<BatchSpanProcessor> | undefined;\n\n  /**\n   * A set of spans used to track spans that we care about, so that we can\n   * export them to the OTel endpoint.\n   *\n   * If a span falls out of reference, it will be removed from this set as we'll\n   * never get a chance to export it or remove it anyway.\n   */\n  #spansToExport = new WeakSet<Span>();\n\n  /**\n   * A map of span IDs to their parent state, which includes a block of\n   * information that can be used and pushed back to the Inngest endpoint to\n   * ingest spans.\n   */\n  #traceParents = new Map<string, ParentState>();\n\n  /**\n   * A registry used to clean up items from the `traceParents` map when spans\n   * fall out of reference. This is used to avoid memory leaks in the case where\n   * a span is not exported, remains unended, and is left in memory before being\n   * GC'd.\n   */\n  #spanCleanup = new FinalizationRegistry<string>((spanId) => {\n    if (spanId) {\n      this.#traceParents.delete(spanId);\n    }\n  });\n\n  /**\n   * Tracks the currently-executing step for each execution, keyed by root span\n   * ID. Used to compute deterministic parent span IDs for userland spans when\n   * checkpointing is enabled and multiple steps run in a single invocation.\n   */\n  #activeStepContext = new Map<\n    string,\n    { hashedStepId: string; attempt: number }\n  >();\n\n  /**\n   * Root span IDs that have had at least one step execution declared, meaning\n   * they are checkpointing runs. Used to filter out infrastructure spans\n   * (checkpoint POSTs, dev server polls) that fire between steps.\n   */\n  #checkpointingRoots = new Set<string>();\n\n  /**\n   * In order to only capture a subset of spans, we need to declare the initial\n   * span that we care about and then export its children.\n   *\n   * Call this method (ideally just before execution starts) with that initial\n   * span to trigger capturing all following children as well as initialize the\n   * batcher.\n   */\n  public declareStartingSpan({\n    span,\n    runId,\n    traceparent,\n    tracestate,\n  }: {\n    span: Span;\n    runId: string;\n    traceparent: string | undefined;\n    tracestate: string | undefined;\n  }): void {\n    // Upsert the batcher ready for later. We do this here to bootstrap it with\n    // the correct async context as soon as we can. As this method is only\n    // called just before execution, we know we're all set up.\n    //\n    // Waiting to call this until we actually need the batcher would mean that\n    // we might not have the correct async context set up, as we'd likely be in\n    // some span lifecycle method that doesn't have the same chain of execution.\n    void this.ensureBatcherInitialized();\n\n    // If we don't have a traceparent, then we can't track this span. This is\n    // likely a span that we don't care about, so we can ignore it.\n    if (!traceparent) {\n      return processorDevDebug(\n        \"no traceparent found for span\",\n        span.spanContext().spanId,\n        \"so skipping it\",\n      );\n    }\n\n    // We also attempt to use `tracestate`. The values we fetch from these\n    // should be optional, as it's likely the Executor won't need us to parrot\n    // them back in later versions.\n    let appId: string | undefined;\n    let functionId: string | undefined;\n    let traceRef: string | undefined;\n\n    if (tracestate) {\n      try {\n        const entries = Object.fromEntries(\n          tracestate.split(\",\").map((kv) => kv.split(\"=\") as [string, string]),\n        );\n\n        appId = entries[TraceStateKey.AppId];\n        functionId = entries[TraceStateKey.FunctionId];\n        traceRef = entries[TraceStateKey.TraceRef];\n      } catch (err) {\n        processorDevDebug(\n          \"failed to parse tracestate\",\n          tracestate,\n          \"so skipping it;\",\n          err,\n        );\n      }\n    }\n\n    // This is a span that we care about, so let's make sure it and its\n    // children are exported.\n    processorDevDebug.extend(\"declareStartingSpan\")(\n      \"declaring:\",\n      span.spanContext().spanId,\n      \"for traceparent\",\n      traceparent,\n    );\n\n    // Set a load of attributes on this span so that we can nicely identify\n    // runtime, paths, etc. Only this span will have these attributes.\n    span.setAttributes(InngestSpanProcessor.resourceAttributes.attributes);\n\n    this.trackSpan(\n      {\n        appId,\n        functionId,\n        runId,\n        traceparent,\n        traceRef,\n        rootSpanId: span.spanContext().spanId,\n      },\n      span,\n      true,\n    );\n  }\n\n  /**\n   * Declare that a step is currently executing. Userland spans created while\n   * a step context is active will have their `inngest.traceparent` rewritten\n   * to reference a deterministic span ID derived from the step, matching the\n   * span the Go executor will create via checkpoint.\n   */\n  public declareStepExecution(\n    rootSpanId: string,\n    hashedStepId: string,\n    attempt: number,\n  ): void {\n    processorDevDebug(\n      \"declareStepExecution: rootSpanId=%s hashedStepId=%s attempt=%d\",\n      rootSpanId,\n      hashedStepId,\n      attempt,\n    );\n    this.#checkpointingRoots.add(rootSpanId);\n    this.#activeStepContext.set(rootSpanId, { hashedStepId, attempt });\n  }\n\n  /**\n   * Clear the active step context after a step finishes executing.\n   */\n  public clearStepExecution(rootSpanId: string): void {\n    processorDevDebug(\"clearStepExecution: rootSpanId=%s\", rootSpanId);\n    this.#activeStepContext.delete(rootSpanId);\n  }\n\n  /**\n   * A getter for retrieving resource attributes for the current process. This\n   * is used to set the resource attributes for the spans that are exported to\n   * the Inngest endpoint, and cache them for later use.\n   */\n  static get resourceAttributes(): Resource {\n    if (!_resourceAttributes) {\n      _resourceAttributes = detectResources({\n        detectors: [\n          osDetector,\n          envDetector,\n          hostDetector,\n          processDetector,\n          serviceInstanceIdDetector,\n        ],\n      });\n    }\n\n    return _resourceAttributes;\n  }\n\n  /**\n   * The batcher is a singleton that is used to export spans to the OTel\n   * endpoint. It is created lazily to avoid creating it until the Inngest App\n   * has been initialized and has had a chance to receive environment variables,\n   * which may be from an incoming request.\n   *\n   * The batcher is only referenced once we've found a span we're interested in,\n   * so this should always have everything it needs on the app by then.\n   */\n  private ensureBatcherInitialized(): Promise<BatchSpanProcessor> {\n    if (!this.#batcher) {\n      this.#batcher = new Promise(async (resolve, reject) => {\n        try {\n          // We retrieve the app from the async context, so we must make sure\n          // that this function is called from the correct chain.\n          const store = await getAsyncCtx();\n          if (!store) {\n            throw new Error(\n              \"No async context found; cannot create batcher to export traces\",\n            );\n          }\n\n          const app = store.app as Inngest.Any;\n\n          const path = \"/v1/traces/userland\";\n          const url = new URL(path, app.apiBaseUrl);\n\n          processorDevDebug(\n            \"batcher lazily accessed; creating new batcher with URL\",\n            url,\n          );\n\n          const exporter = new OTLPTraceExporter({\n            url: url.href,\n            headers: {\n              ...app.headers,\n              Authorization: `Bearer ${app.signingKey}`,\n            },\n          });\n\n          resolve(new BatchSpanProcessor(exporter));\n        } catch (err) {\n          reject(err);\n        }\n      });\n    }\n\n    return this.#batcher;\n  }\n\n  /**\n   * Mark a span as being tracked by this processor, meaning it will be exported\n   * to the Inggest endpoint when it ends.\n   */\n  private trackSpan(\n    parentState: ParentState,\n    span: Span,\n    isRoot = false,\n  ): void {\n    const trackDebug = processorDevDebug.extend(\"trackSpan\");\n    const spanId = span.spanContext().spanId;\n\n    this.#spanCleanup.register(span, spanId, span);\n    this.#spansToExport.add(span);\n    this.#traceParents.set(spanId, parentState);\n\n    // For direct children of the root span during step execution, set a\n    // dedicated attribute with the deterministic step span ID. The Go executor\n    // creates executor.step spans with the same deterministic ID (from the same\n    // seed), so the ingestion can parent userland spans under the correct step.\n    if (!isRoot) {\n      const spanParentId =\n        (span as unknown as ReadableSpan).parentSpanContext?.spanId ??\n        (span as unknown as { parentSpanId?: string }).parentSpanId;\n\n      if (spanParentId === parentState.rootSpanId) {\n        const stepCtx = this.#activeStepContext.get(parentState.rootSpanId);\n        if (stepCtx) {\n          const seed = stepCtx.hashedStepId + \":\" + String(stepCtx.attempt);\n          const newSpanId = deterministicSpanID(seed);\n          trackDebug(\n            \"setting inngest.step.parentSpanId=%s (seed=%s) on span %s\",\n            newSpanId,\n            seed,\n            spanId,\n          );\n          span.setAttribute(Attribute.InngestStepParentSpanId, newSpanId);\n        }\n      }\n    }\n\n    span.setAttribute(Attribute.InngestTraceparent, parentState.traceparent);\n    span.setAttribute(Attribute.InngestRunId, parentState.runId);\n\n    // Setting app ID is optional; it's likely in future versions of the\n    // Executor that we don't need to parrot this back.\n    if (parentState.appId) {\n      span.setAttribute(Attribute.InngestAppId1, parentState.appId);\n      span.setAttribute(Attribute.InngestAppId2, parentState.appId);\n    }\n\n    // Setting function ID is optional; it's likely in future versions of the\n    // Executor that we don't need to parrot this back.\n    if (parentState.functionId) {\n      span.setAttribute(Attribute.InngestFunctionId, parentState.functionId);\n    }\n\n    if (parentState.traceRef) {\n      span.setAttribute(Attribute.InngestTraceRef, parentState.traceRef);\n    }\n  }\n\n  /**\n   * Clean up any references to a span that has ended. This is used to avoid\n   * memory leaks in the case where a span is not exported, remains unended, and\n   * is left in memory before being GC'd.\n   */\n  private cleanupSpan(span: Span): void {\n    const spanId = span.spanContext().spanId;\n\n    // This span is no longer in use, so we can remove it from the cleanup\n    // registry.\n    this.#spanCleanup.unregister(span);\n    this.#spansToExport.delete(span);\n    this.#traceParents.delete(spanId);\n  }\n\n  /**\n   * An implementation of the `onStart` method from the `SpanProcessor`\n   * interface. This is called when a span is started, and is used to track\n   * spans that are children of spans we care about.\n   */\n  onStart(span: Span): void {\n    const devDebug = processorDevDebug.extend(\"onStart\");\n    const spanId = span.spanContext().spanId;\n    // Support both OTel SDK v2.x (parentSpanContext.spanId) and v1.x\n    // (parentSpanId as a plain string) since users may have either version.\n    const parentSpanId =\n      (span as unknown as ReadableSpan).parentSpanContext?.spanId ??\n      (span as unknown as { parentSpanId?: string }).parentSpanId;\n\n    // The root span isn't captured here, but we can capture children of it\n    // here.\n\n    if (!parentSpanId) {\n      // All spans that Inngest cares about will have a parent, so ignore this\n      devDebug(\"no parent span ID for\", spanId, \"so skipping it\");\n\n      return;\n    }\n\n    const parentState = this.#traceParents.get(parentSpanId);\n    if (parentState) {\n      // In checkpointing mode, only track spans during active step execution.\n      // This filters out infrastructure spans (checkpoint POSTs, dev server\n      // polls) that fire between steps and would otherwise pollute the tree.\n      if (\n        this.#checkpointingRoots.has(parentState.rootSpanId) &&\n        !this.#activeStepContext.has(parentState.rootSpanId)\n      ) {\n        processorDevDebug(\n          \"skipping span\",\n          spanId,\n          \"- checkpointing between steps\",\n        );\n        return;\n      }\n\n      // This span is a child of a span we care about, so add it to the list of\n      // tracked spans so that we also capture its children\n      devDebug(\n        \"found traceparent\",\n        parentState,\n        \"in span ID\",\n        parentSpanId,\n        \"so adding\",\n        spanId,\n      );\n\n      this.trackSpan(parentState, span);\n    }\n  }\n\n  /**\n   * An implementation of the `onEnd` method from the `SpanProcessor` interface.\n   * This is called when a span ends, and is used to export spans to the Inngest\n   * endpoint.\n   */\n  onEnd(span: ReadableSpan): void {\n    const devDebug = processorDevDebug.extend(\"onEnd\");\n    const spanId = span.spanContext().spanId;\n\n    try {\n      if (this.#spansToExport.has(span as unknown as Span)) {\n        if (!this.#batcher) {\n          return devDebug(\n            \"batcher not initialized, so failed exporting span\",\n            spanId,\n          );\n        }\n\n        devDebug(\"exporting span\", spanId);\n        return void this.#batcher.then((batcher) => batcher.onEnd(span));\n      }\n\n      devDebug(\"not exporting span\", spanId, \"as we don't care about it\");\n    } finally {\n      this.cleanupSpan(span as unknown as Span);\n    }\n  }\n\n  /**\n   * An implementation of the `forceFlush` method from the `SpanProcessor`\n   * interface. This is called to force the processor to flush any spans that\n   * are currently in the batcher. This is used to ensure that spans are\n   * exported to the Inngest endpoint before the process exits.\n   *\n   * Notably, we call this in the `wrapRequest` middleware hook to ensure\n   * that spans for a run as exported as soon as possible and before the\n   * serverless process is killed.\n   */\n  async forceFlush(): Promise<void> {\n    const flushDebug = processorDevDebug.extend(\"forceFlush\");\n    flushDebug(\"force flushing batcher\");\n\n    return this.#batcher\n      ?.then((batcher) => batcher.forceFlush())\n      .catch((err) => {\n        flushDebug(\"error flushing batcher\", err, \"ignoring\");\n      });\n  }\n\n  async shutdown(): Promise<void> {\n    processorDevDebug.extend(\"shutdown\")(\"shutting down batcher\");\n\n    return this.#batcher?.then((batcher) => batcher.shutdown());\n  }\n}\n\n/**\n * An OTel span processor that is used to export spans to the Inngest endpoint.\n * This is used to track spans that are created within an Inngest run and export\n * them to the Inngest endpoint for tracing.\n *\n * It's careful to only pick relevant spans to export and will not send any\n * irrelevant spans to the Inngest endpoint.\n */\nexport class PublicInngestSpanProcessor extends InngestSpanProcessor {\n  constructor(\n    /**\n     * The app that this span processor is associated with. This is used to\n     * determine the Inngest endpoint to export spans to.\n     */\n    app: Inngest.Like,\n  ) {\n    super(app);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAuBA,MAAM,uCAA0B,GAAGA,2BAAY,uBAAuB;;;;;;;AAQtE,IAAIC;;;;;;;;;;;;;AA2BJ,IAAa,uBAAb,MAAa,qBAA8C;;;;;;;;;CASzD,YAYE,KACA;AACA,MAAI,IACF,mCAAmB,IAAI,KAAoB,KAAK;;;;;;;;CAUpD;;;;;;;;CASA,iCAAiB,IAAI,SAAe;;;;;;CAOpC,gCAAgB,IAAI,KAA0B;;;;;;;CAQ9C,eAAe,IAAI,sBAA8B,WAAW;AAC1D,MAAI,OACF,OAAKC,aAAc,OAAO,OAAO;GAEnC;;;;;;CAOF,qCAAqB,IAAI,KAGtB;;;;;;CAOH,sCAAsB,IAAI,KAAa;;;;;;;;;CAUvC,AAAO,oBAAoB,EACzB,MACA,OACA,aACA,cAMO;AAQP,EAAK,KAAK,0BAA0B;AAIpC,MAAI,CAAC,YACH,QAAO,kBACL,iCACA,KAAK,aAAa,CAAC,QACnB,iBACD;EAMH,IAAIC;EACJ,IAAIC;EACJ,IAAIC;AAEJ,MAAI,WACF,KAAI;GACF,MAAM,UAAU,OAAO,YACrB,WAAW,MAAM,IAAI,CAAC,KAAK,OAAO,GAAG,MAAM,IAAI,CAAqB,CACrE;AAED,WAAQ,QAAQC,6BAAc;AAC9B,gBAAa,QAAQA,6BAAc;AACnC,cAAW,QAAQA,6BAAc;WAC1B,KAAK;AACZ,qBACE,8BACA,YACA,mBACA,IACD;;AAML,oBAAkB,OAAO,sBAAsB,CAC7C,cACA,KAAK,aAAa,CAAC,QACnB,mBACA,YACD;AAID,OAAK,cAAc,qBAAqB,mBAAmB,WAAW;AAEtE,OAAK,UACH;GACE;GACA;GACA;GACA;GACA;GACA,YAAY,KAAK,aAAa,CAAC;GAChC,EACD,MACA,KACD;;;;;;;;CASH,AAAO,qBACL,YACA,cACA,SACM;AACN,oBACE,kEACA,YACA,cACA,QACD;AACD,QAAKC,mBAAoB,IAAI,WAAW;AACxC,QAAKC,kBAAmB,IAAI,YAAY;GAAE;GAAc;GAAS,CAAC;;;;;CAMpE,AAAO,mBAAmB,YAA0B;AAClD,oBAAkB,qCAAqC,WAAW;AAClE,QAAKA,kBAAmB,OAAO,WAAW;;;;;;;CAQ5C,WAAW,qBAA+B;AACxC,MAAI,CAAC,oBACH,sEAAsC,EACpC,WAAW;GACTC;GACAC;GACAC;GACAC;GACAC;GACD,EACF,CAAC;AAGJ,SAAO;;;;;;;;;;;CAYT,AAAQ,2BAAwD;AAC9D,MAAI,CAAC,MAAKC,QACR,OAAKA,UAAW,IAAI,QAAQ,OAAO,SAAS,WAAW;AACrD,OAAI;IAGF,MAAM,QAAQ,MAAMC,yBAAa;AACjC,QAAI,CAAC,MACH,OAAM,IAAI,MACR,iEACD;IAGH,MAAM,MAAM,MAAM;IAGlB,MAAM,MAAM,IAAI,IADH,uBACa,IAAI,WAAW;AAEzC,sBACE,0DACA,IACD;AAUD,YAAQ,IAAIC,kDARK,IAAIC,2DAAkB;KACrC,KAAK,IAAI;KACT,SAAS;MACP,GAAG,IAAI;MACP,eAAe,UAAU,IAAI;MAC9B;KACF,CAAC,CAEsC,CAAC;YAClC,KAAK;AACZ,WAAO,IAAI;;IAEb;AAGJ,SAAO,MAAKH;;;;;;CAOd,AAAQ,UACN,aACA,MACA,SAAS,OACH;EACN,MAAM,aAAa,kBAAkB,OAAO,YAAY;EACxD,MAAM,SAAS,KAAK,aAAa,CAAC;AAElC,QAAKI,YAAa,SAAS,MAAM,QAAQ,KAAK;AAC9C,QAAKC,cAAe,IAAI,KAAK;AAC7B,QAAKjB,aAAc,IAAI,QAAQ,YAAY;AAM3C,MAAI,CAAC,QAKH;QAHG,KAAiC,mBAAmB,UACpD,KAA8C,kBAE5B,YAAY,YAAY;IAC3C,MAAM,UAAU,MAAKM,kBAAmB,IAAI,YAAY,WAAW;AACnE,QAAI,SAAS;KACX,MAAM,OAAO,QAAQ,eAAe,MAAM,OAAO,QAAQ,QAAQ;KACjE,MAAM,YAAYY,4CAAoB,KAAK;AAC3C,gBACE,6DACA,WACA,MACA,OACD;AACD,UAAK,aAAaC,yBAAU,yBAAyB,UAAU;;;;AAKrE,OAAK,aAAaA,yBAAU,oBAAoB,YAAY,YAAY;AACxE,OAAK,aAAaA,yBAAU,cAAc,YAAY,MAAM;AAI5D,MAAI,YAAY,OAAO;AACrB,QAAK,aAAaA,yBAAU,eAAe,YAAY,MAAM;AAC7D,QAAK,aAAaA,yBAAU,eAAe,YAAY,MAAM;;AAK/D,MAAI,YAAY,WACd,MAAK,aAAaA,yBAAU,mBAAmB,YAAY,WAAW;AAGxE,MAAI,YAAY,SACd,MAAK,aAAaA,yBAAU,iBAAiB,YAAY,SAAS;;;;;;;CAStE,AAAQ,YAAY,MAAkB;EACpC,MAAM,SAAS,KAAK,aAAa,CAAC;AAIlC,QAAKH,YAAa,WAAW,KAAK;AAClC,QAAKC,cAAe,OAAO,KAAK;AAChC,QAAKjB,aAAc,OAAO,OAAO;;;;;;;CAQnC,QAAQ,MAAkB;EACxB,MAAM,WAAW,kBAAkB,OAAO,UAAU;EACpD,MAAM,SAAS,KAAK,aAAa,CAAC;EAGlC,MAAM,eACH,KAAiC,mBAAmB,UACpD,KAA8C;AAKjD,MAAI,CAAC,cAAc;AAEjB,YAAS,yBAAyB,QAAQ,iBAAiB;AAE3D;;EAGF,MAAM,cAAc,MAAKA,aAAc,IAAI,aAAa;AACxD,MAAI,aAAa;AAIf,OACE,MAAKK,mBAAoB,IAAI,YAAY,WAAW,IACpD,CAAC,MAAKC,kBAAmB,IAAI,YAAY,WAAW,EACpD;AACA,sBACE,iBACA,QACA,gCACD;AACD;;AAKF,YACE,qBACA,aACA,cACA,cACA,aACA,OACD;AAED,QAAK,UAAU,aAAa,KAAK;;;;;;;;CASrC,MAAM,MAA0B;EAC9B,MAAM,WAAW,kBAAkB,OAAO,QAAQ;EAClD,MAAM,SAAS,KAAK,aAAa,CAAC;AAElC,MAAI;AACF,OAAI,MAAKW,cAAe,IAAI,KAAwB,EAAE;AACpD,QAAI,CAAC,MAAKL,QACR,QAAO,SACL,qDACA,OACD;AAGH,aAAS,kBAAkB,OAAO;AAC3B,IAAK,MAAKA,QAAS,MAAM,YAAY,QAAQ,MAAM,KAAK,CAAC;AAAhE;;AAGF,YAAS,sBAAsB,QAAQ,4BAA4B;YAC3D;AACR,QAAK,YAAY,KAAwB;;;;;;;;;;;;;CAc7C,MAAM,aAA4B;EAChC,MAAM,aAAa,kBAAkB,OAAO,aAAa;AACzD,aAAW,yBAAyB;AAEpC,SAAO,MAAKA,SACR,MAAM,YAAY,QAAQ,YAAY,CAAC,CACxC,OAAO,QAAQ;AACd,cAAW,0BAA0B,KAAK,WAAW;IACrD;;CAGN,MAAM,WAA0B;AAC9B,oBAAkB,OAAO,WAAW,CAAC,wBAAwB;AAE7D,SAAO,MAAKA,SAAU,MAAM,YAAY,QAAQ,UAAU,CAAC;;;;;;;;;;;AAY/D,IAAa,6BAAb,cAAgD,qBAAqB;CACnE,YAKE,KACA;AACA,QAAM,IAAI"}