snowflake-sdk/dist/lib/file_transfer_agent/gcs_util.d.ts

Node.js driver for Snowflake

export = GCSUtil;
/**
 * How GCS PUT/GET works (read this first)
 * =======================================
 * A transfer happens in two phases:
 *   1. Resolve  - the driver sends the PUT/GET SQL to the Snowflake server (GS),
 *                 which decides where the bytes go and how the client may write
 *                 there, then returns a `stageInfo` (bucket, path prefix,
 *                 credentials, and sometimes a presigned URL).
 *   2. Transfer - the driver streams bytes straight to the GCS bucket using what
 *                 GS returned. The driver never uploads to Snowflake directly.
 *
 * For GCS, GS returns exactly ONE of two credential styles per session (never a
 * mix):
 *   (A) Downscoped access token (`stageInfo.creds.GCS_ACCESS_TOKEN`) - PREFERRED.
 *       "Downscoped" means folder/prefix-scoped, NOT object-scoped: one token
 *       authorizes every object under the staging path, so a single token covers
 *       all files in the PUT.
 *   (B) Presigned URL - FALLBACK/legacy. A signed URL authorizing one operation
 *       on exactly one object, so you need one presigned URL per file. These are
 *       minted by `updateFileMetasWithPresignedUrl` in file_transfer_agent.js,
 *       which re-resolves the PUT once per file (see the put-rank warning there).
 *
 * How each style authorizes a request at transfer time:
 *   - Token mode:     PUT/GET `generateFileURL(stageInfo)` + file name, with an
 *                     `Authorization: Bearer <GCS_ACCESS_TOKEN>` header.
 *   - Presigned mode: PUT/GET the presigned URL itself; no Authorization header.
 *   In token mode the URL and the token MUST come from the same resolution.
 *
 * Landmarks in this file: `createClient` is the mode switch; `uploadFile` picks
 * the upload path; `applyGcsUploadError` maps HTTP failures onto the retry/renew
 * states the outer loop understands (401 -> renew token, 400 -> renew presigned
 * URL).
 */
/**
 * Creates an GCS utility object.
 * @param {module} connectionConfig
 * @param {module} httpClient
 * @param {module} fileStream
 *
 * @returns {Object}
 * @constructor
 */
declare function GCSUtil(connectionConfig: any, httpClient: any): Object;
declare class GCSUtil {
    /**
     * How GCS PUT/GET works (read this first)
     * =======================================
     * A transfer happens in two phases:
     *   1. Resolve  - the driver sends the PUT/GET SQL to the Snowflake server (GS),
     *                 which decides where the bytes go and how the client may write
     *                 there, then returns a `stageInfo` (bucket, path prefix,
     *                 credentials, and sometimes a presigned URL).
     *   2. Transfer - the driver streams bytes straight to the GCS bucket using what
     *                 GS returned. The driver never uploads to Snowflake directly.
     *
     * For GCS, GS returns exactly ONE of two credential styles per session (never a
     * mix):
     *   (A) Downscoped access token (`stageInfo.creds.GCS_ACCESS_TOKEN`) - PREFERRED.
     *       "Downscoped" means folder/prefix-scoped, NOT object-scoped: one token
     *       authorizes every object under the staging path, so a single token covers
     *       all files in the PUT.
     *   (B) Presigned URL - FALLBACK/legacy. A signed URL authorizing one operation
     *       on exactly one object, so you need one presigned URL per file. These are
     *       minted by `updateFileMetasWithPresignedUrl` in file_transfer_agent.js,
     *       which re-resolves the PUT once per file (see the put-rank warning there).
     *
     * How each style authorizes a request at transfer time:
     *   - Token mode:     PUT/GET `generateFileURL(stageInfo)` + file name, with an
     *                     `Authorization: Bearer <GCS_ACCESS_TOKEN>` header.
     *   - Presigned mode: PUT/GET the presigned URL itself; no Authorization header.
     *   In token mode the URL and the token MUST come from the same resolution.
     *
     * Landmarks in this file: `createClient` is the mode switch; `uploadFile` picks
     * the upload path; `applyGcsUploadError` maps HTTP failures onto the retry/renew
     * states the outer loop understands (401 -> renew token, 400 -> renew presigned
     * URL).
     */
    /**
     * Creates an GCS utility object.
     * @param {module} connectionConfig
     * @param {module} httpClient
     * @param {module} fileStream
     *
     * @returns {Object}
     * @constructor
     */
    constructor(connectionConfig: any, httpClient: any);
    /**
     * Build the per-session GCS client, which is really just the credential-style
     * switch (see the module overview at the top of this file):
     *   - token mode:     returns `{ gcsToken }` from `stageInfo.creds.GCS_ACCESS_TOKEN`
     *   - presigned mode: returns `null` (no token; each meta carries its own
     *                     presigned URL instead)
     * Also configures the HTTP client for the resolved GCS endpoint.
     *
     * @param {Object} stageInfo
     *
     * @returns {?{gcsToken: string}} token-bearing client, or null in presigned mode
     */
    createClient: (stageInfo: Object) => {
        gcsToken: string;
    } | null;
    /**
     * Extract the bucket name and path from the metadata's stage location.
     *
     * @param {String} stageLocation
     *
     * @returns {GCSLocation}
     */
    extractBucketNameAndPath: (stageLocation: string) => GCSLocation;
    /**
     * Create file header based on file being uploaded or not.
     *
     * @param {Object} meta
     * @param {String} filename
     *
     * @returns {Object}
     */
    getFileHeader: (meta: Object, filename: string) => Object;
    /**
     * Read the file's stat, then dispatch to one of two upload paths. Which path
     * applies depends on the credential style (token vs presigned; see the module
     * overview) and the file size:
     *
     *   1. **Single Buffer-bodied PUT** — when the session uses a legacy
     *      presigned URL or the file is no larger than
     *      `MULTIPART_THRESHOLD_BYTES`. Workspace stage presigned URLs are
     *      signed for one specific PUT and cannot initiate a resumable upload.
     *      So when GS hands back a `presignedUrl` (the legacy path;
     *      pre-CB_2023_06 deployments or driver versions before 1.6.21), large
     *      files fall back to a single Buffer PUT even though the access-token
     *      path would split into chunks.
     *   2. **XML API resumable upload session** — when an access token is
     *      available and the file exceeds `MULTIPART_THRESHOLD_BYTES`. Bounds
     *      in-flight memory by `MULTIPART_PART_SIZE_BYTES` and gives per-chunk
     *      retry granularity. See `uploadFileResumable` for protocol
     *      details.
     *
     * @param {String} dataFile
     * @param {Object} meta
     * @param {Object} encryptionMetadata
     * @param {Number} maxConcurrency
     */
    uploadFile: (dataFile: string, meta: Object, encryptionMetadata: Object, maxConcurrency: number) => Promise<void>;
    /**
     * GCS resumable upload session for `dataFile` (access-token mode only;
     * presigned URLs cannot initiate a resumable session). Reads
     * `MULTIPART_PART_SIZE_BYTES` bytes per chunk into a fresh Buffer and PUTs
     * each as a `Content-Range`-tagged chunk against the session URL minted by
     * the initiation POST. On transient chunk failure, queries the session for
     * the committed offset and resumes from there. On terminal failure,
     * best-effort DELETE the session so the upload doesn't linger as a
     * half-staged blob.
     *
     * Uses GCS's XML API resumable variant (initiate POST against the same
     * bucket-path URL the single-PUT uses, with `x-goog-resumable: start`)
     * rather than JSON API resumable, since GS hands the connector that XML
     * shape; honoring it avoids inventing a path prefix that the server
     * never returned and keeps strict-allowlisting environments working.
     *
     * @param {String} dataFile
     * @param {Number} fileSize
     * @param {Object} meta
     * @param {Object} encryptionMetadata
     */
    uploadFileResumable: (dataFile: string, fileSize: number, meta: Object, encryptionMetadata: Object) => Promise<void>;
    /**
     * Single-PUT upload used by both credential styles (and the small-file /
     * legacy fallback when a resumable session isn't used). Picks the destination
     * the same way every GCS request does:
     *   - presigned mode: PUT to `meta.presignedUrl` (no Authorization header)
     *   - token mode:     PUT to `generateFileURL(meta.stageInfo)` with a Bearer
     *                     token from `meta.client.gcsToken`
     * In token mode the URL (from `meta.stageInfo`) and the token (from
     * `meta.client`) must originate from the same resolution; see the put-rank
     * warning in `updateFileMetasWithPresignedUrl` for why mixing resolutions
     * yields a 403.
     *
     * @param {Buffer|string|stream.Readable} fileStream
     * @param {Object} meta
     * @param {Object} encryptionMetadata
     *
     * @returns {null}
     */
    uploadFileStream: (fileStream: Buffer | string | stream.Readable, meta: Object, encryptionMetadata: Object) => null;
    /**
     * Download the file.
     *
     * @param {Object} meta
     * @param fullDstPath
     *
     * @returns {null}
     */
    nativeDownloadFile: (meta: Object, fullDstPath: any) => null;
    /**
     * Generate file URL based on bucket.
     *
     * @param {Object} stageInfo
     * @param {String} filename
     *
     * @returns {String}
     */
    generateFileURL: (stageInfo: Object, filename: string) => string;
    getGCSCustomEndPoint: (stageInfo: any) => any;
    setupHttpClient: (endPoint: any) => void;
}
declare namespace GCSUtil {
    export { GCSLocation };
}
type GCSLocation = {
    bucketName: string;
    path: string;
};