@juspay/neurolink/dist/adapters/imageFormatSupport.d.ts

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Vision-provider image format compatibility.
 *
 * NeuroLink identifies far more image formats than any vision API accepts. BMP,
 * TIFF, AVIF, ICO and JPEG 2000 are accepted by none of them; HEIC and HEIF are
 * accepted only by Google. Until this module existed those formats were
 * detected correctly, labelled correctly, and then forwarded verbatim — the
 * request reached the provider and came back as an opaque HTTP 400, which is
 * the least useful outcome available: the file was clearly an image, NeuroLink
 * knew exactly which kind, and still nothing worked.
 *
 * A phone photo is the common case. iOS writes HEIC by default, so "attach a
 * photo and ask what is in it" failed for every provider except Google.
 *
 * Anything outside the universal set is transcoded to PNG. PNG rather than JPEG
 * because the sources are frequently lossless (TIFF, BMP, ICO) or already
 * carry alpha, and a lossy re-encode of an image the model is about to read
 * closely is the wrong default.
 *
 * @module adapters/imageFormatSupport
 */
import type { ExifOrientationProbe, ImageOrientationNormalization, ImageWithAltText, VisionImageConversion, VisionTranscodeOptions } from "../types/index.js";
/**
 * MIME types every vision-capable provider accepts as-is.
 *
 * This is the intersection across OpenAI, Anthropic, Google (AI Studio and
 * Vertex), Bedrock, Azure and Mistral — deliberately the intersection and not
 * a per-provider matrix. Google additionally accepts HEIC/HEIF natively, but
 * converting those for Google as well costs one transcode and removes an
 * entire axis of provider-specific branching from the dispatch path.
 */
export declare const UNIVERSAL_VISION_IMAGE_MIME_TYPES: ReadonlySet<string>;
/**
 * Every image MIME type NeuroLink accepts as *input*.
 *
 * The union of what providers take as-is and what this module can convert for
 * them. Intake validation must use this rather than the universal set alone:
 * a format we can transcode is a format we accept, and gating intake on the
 * provider-acceptable list rejects the file before conversion ever runs.
 */
export declare const SUPPORTED_INPUT_IMAGE_MIME_TYPES: ReadonlySet<string>;
/**
 * True when a MIME type needs transcoding before it can be sent to a vision
 * provider. Cheap enough to call on every image; callers use it to avoid
 * reading a file off disk that would not have been converted anyway.
 */
export declare function needsVisionTranscode(mimeType: string): boolean;
/**
 * True when a MIME type's format can carry an orientation tag at all. Cheap
 * enough to call on every image; callers use it to skip reading a file off
 * disk (or invoking sharp) for a format that structurally cannot need it.
 */
export declare function mayCarryExifOrientation(mimeType: string): boolean;
/**
 * Leading bytes {@link probeExifOrientation} needs to reach a verdict on the
 * containers it parses.
 *
 * A JPEG's EXIF block lives in an APP1 segment whose length field is 16 bits
 * wide, so the segment cannot exceed 64 KiB — and IFD0, which holds the
 * orientation tag, sits at the very front of it. One window this size
 * therefore spans the entire tag block of any file written EXIF-first, which
 * is every file a camera or phone produces. WebP settles in 21 bytes; PNG
 * walks complete chunks through the first IDAT and falls back if this window
 * ends first.
 *
 * Sized deliberately in terms of what the format guarantees rather than what
 * a sample of files happens to need: a window chosen by measurement would
 * turn an unusual-but-legal layout into a silent wrong answer, whereas
 * running out of this one returns `"inconclusive"` and costs only the read it
 * was trying to avoid.
 */
export declare const EXIF_PROBE_PREFIX_BYTES: number;
/**
 * Decide from a bounded header prefix whether an image carries an EXIF
 * orientation worth acting on — without decoding it, and without needing all
 * of its bytes.
 *
 * This exists because the orientation question is asked of every JPEG, PNG
 * and WebP a caller supplies, while the answer is "no" for almost all of them.
 * Answering it with a decoder means reading the whole file into memory and
 * loading sharp to look at a header; answering it here costs
 * {@link EXIF_PROBE_PREFIX_BYTES} at most, and usually far less.
 *
 * The verdict is deliberately three-valued. `"absent"` is a promise — the
 * container was parsed far enough to rule an actionable tag out — and callers
 * may skip the expensive path on it. Everything else, including every
 * container this probe does not parse (HEIC, HEIF, AVIF: ISOBMFF, where
 * orientation is an item property rather than a header field), comes back
 * `"inconclusive"` and must fall through to the decoder.
 *
 * @param prefix - Leading bytes of the image; the whole image is also fine.
 * @param mimeType - Detected MIME type of the image `prefix` came from.
 */
export declare function probeExifOrientation(prefix: Buffer, mimeType: string): ExifOrientationProbe;
/**
 * Return image bytes every vision provider can read, transcoding to PNG when
 * the source format is one no provider accepts.
 *
 * Never throws for image reasons. When neither backend can decode the input,
 * the original bytes are returned with a warning naming the format — the
 * request then fails at the provider exactly as it did before, rather than this
 * compatibility step becoming a new way for a previously working request to
 * break.
 *
 * `options.autoOrient` folds an EXIF orientation correction into the same
 * decode, for a format that needs both (HEIC, HEIF, TIFF, AVIF are in both
 * sets). The sharp backend honours it explicitly, via `.rotate()`.
 *
 * The ffmpeg backends take no flag for it, and `-autorotate` would only
 * restate a default rather than fix anything — but that is established for
 * one orientation mechanism only, so read the scope before relying on it.
 *
 * **Measured — EXIF-IFD orientation.** A 300x150 JPEG tagged `Orientation=6`
 * came back 150x300 with the tag consumed on system ffmpeg 8.1.1 and on the
 * bundled `ffmpeg-static` 6.0, and stayed 300x150 on both only when
 * `-noautorotate` was forced, which is the control showing the default did
 * the work. TIFF stores orientation the same way.
 *
 * ⚠️ **Not measured — ISOBMFF `irot`/`imir`.** HEIC, HEIF and AVIF carry
 * orientation as item properties, a structurally different mechanism the
 * EXIF result says nothing about. That gap matters more than its size
 * suggests: HEVC-coded HEIC is the main reason this fallback exists at all,
 * since sharp cannot decode it, whereas the JPEG used above never reaches
 * ffmpeg in production — JPEG is universal and is never transcoded, so the
 * fixture demonstrates the binaries' behaviour rather than a path a request
 * takes. Whether an `irot`-bearing HEIC is corrected here is untested in
 * both directions; treat it as unknown rather than working.
 *
 * `autoOrient` is in any case a request rather than a guarantee, exactly like
 * `converted`: a source no backend can decode is returned untouched.
 *
 * ⚠️ Two further things would quietly break the measured case, and neither
 * fails a test, because the fused path reports `converted: true` either way.
 * Passing `-noautorotate`, or an ffmpeg build defaulting it off, drops the
 * rotation. And the sharp-first ordering in {@link transcodeBackends} is what
 * keeps the two backends' orientation behaviour equivalent rather than merely
 * similar — sharp strips the tag while ffmpeg's PNG output carries none to
 * begin with.
 *
 * @param buffer - Raw image bytes.
 * @param mimeType - Detected MIME type of `buffer`.
 * @param options - See {@link VisionTranscodeOptions}. `outputFormat`
 *   defaults to `"png"` — see the module header for why that stays the
 *   default for every caller who doesn't opt into `"jpeg"` explicitly.
 */
export declare function toVisionCompatibleImage(buffer: Buffer, mimeType: string, options?: VisionTranscodeOptions): Promise<VisionImageConversion>;
/**
 * Auto-orient an image from its EXIF/XMP tag and strip the tag, so a photo
 * that was stored rotated (the common case for phone cameras, since the
 * sensor is read in one fixed orientation and rotation is recorded as a flag
 * rather than baked into the pixels) is sent upright instead of sideways.
 *
 * Three costs are guarded deliberately, matching how `toVisionCompatibleImage`
 * guards its own:
 * - **Cost**: `mayCarryExifOrientation` skips formats that cannot carry the
 *   tag; `probeExifOrientation` then settles the formats that can from their
 *   own header bytes, so the overwhelmingly common orientation-less JPEG or
 *   WebP never loads sharp at all. Only a container the probe cannot rule
 *   out reaches a `metadata()` call, and only a tag that is present and not
 *   the already-upright value `1` reaches the decode-and-re-encode.
 * - **Availability**: sharp is an optional dependency (the ffmpeg fallback
 *   path above exists precisely because it can be absent). Any failure —
 *   missing package, malformed pixels, a timeout — is caught here and
 *   degrades to returning the original bytes unchanged, never throwing.
 * - **Size**: re-encoding changes byte length. This function only returns the
 *   new bytes; it does not itself re-validate size limits. Callers that
 *   re-encode must re-run their size guard on the returned buffer.
 *
 * Never throws for image reasons — mirrors `toVisionCompatibleImage`.
 *
 * @param buffer - Raw image bytes.
 * @param mimeType - Detected MIME type of `buffer`.
 */
export declare function normalizeImageOrientation(buffer: Buffer, mimeType: string): Promise<ImageOrientationNormalization>;
/**
 * Unwrap an `input.images` entry to its payload.
 *
 * `ImageWithAltText` (`{ data, altText }`) is a documented public input shape,
 * but the provider image loops typed the array as `Buffer | string` and so
 * treated a wrapper as raw bytes — `toString("base64")` on the object yields
 * the literal "[object Object]", which is valid base64 that decodes to seven
 * bytes of garbage. The request therefore reached the API and failed as
 * "invalid image data", with nothing pointing at the stringification.
 *
 * Alt text has no representation in Vertex's `inlineData` part, so it is
 * dropped here deliberately rather than corrupting the payload to carry it.
 */
export declare function unwrapImagePayload(entry: Buffer | string | ImageWithAltText): Buffer | string;