@google-cloud/storage
Version:
Cloud Storage Client Library for Node.js
1,172 lines (1,171 loc) • 137 kB
JavaScript
"use strict";
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
Object.defineProperty(exports, "__esModule", { value: true });
exports.File = exports.FileExceptionMessages = exports.STORAGE_POST_POLICY_BASE_URL = exports.ActionToHTTPMethod = void 0;
const nodejs_common_1 = require("./nodejs-common");
const promisify_1 = require("@google-cloud/promisify");
const compressible = require("compressible");
const getStream = require("get-stream");
const crypto = require("crypto");
const dateFormat = require("date-and-time");
const extend = require("extend");
const fs = require("fs");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const hashStreamValidation = require('hash-stream-validation');
const mime = require("mime");
const os = require("os");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const pumpify = require('pumpify');
const resumableUpload = require("./gcs-resumable-upload");
const stream_1 = require("stream");
const streamEvents = require("stream-events");
const xdgBasedir = require("xdg-basedir");
const zlib = require("zlib");
const storage_1 = require("./storage");
const bucket_1 = require("./bucket");
const acl_1 = require("./acl");
const signer_1 = require("./signer");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const duplexify = require('duplexify');
const util_1 = require("./util");
const retry = require("async-retry");
var ActionToHTTPMethod;
(function (ActionToHTTPMethod) {
ActionToHTTPMethod["read"] = "GET";
ActionToHTTPMethod["write"] = "PUT";
ActionToHTTPMethod["delete"] = "DELETE";
ActionToHTTPMethod["resumable"] = "POST";
})(ActionToHTTPMethod = exports.ActionToHTTPMethod || (exports.ActionToHTTPMethod = {}));
/**
* Custom error type for errors related to creating a resumable upload.
*
* @private
*/
class ResumableUploadError extends Error {
constructor() {
super(...arguments);
this.name = 'ResumableUploadError';
}
}
/**
* @const {string}
* @private
*/
exports.STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com';
/**
* @const {RegExp}
* @private
*/
const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/;
class RequestError extends Error {
}
const SEVEN_DAYS = 7 * 24 * 60 * 60;
var FileExceptionMessages;
(function (FileExceptionMessages) {
FileExceptionMessages["EXPIRATION_TIME_NA"] = "An expiration time is not available.";
FileExceptionMessages["DESTINATION_NO_NAME"] = "Destination file should have a name.";
FileExceptionMessages["INVALID_VALIDATION_FILE_RANGE"] = "Cannot use validation with file ranges (start/end).";
FileExceptionMessages["MD5_NOT_AVAILABLE"] = "MD5 verification was specified, but is not available for the requested object. MD5 is not available for composite objects.";
FileExceptionMessages["EQUALS_CONDITION_TWO_ELEMENTS"] = "Equals condition must be an array of 2 elements.";
FileExceptionMessages["STARTS_WITH_TWO_ELEMENTS"] = "StartsWith condition must be an array of 2 elements.";
FileExceptionMessages["CONTENT_LENGTH_RANGE_MIN_MAX"] = "ContentLengthRange must have numeric min & max fields.";
FileExceptionMessages["DOWNLOAD_MISMATCH"] = "The downloaded data did not match the data from the server. To be sure the content is the same, you should download the file again.";
FileExceptionMessages["UPLOAD_MISMATCH_DELETE_FAIL"] = "The uploaded data did not match the data from the server. \n As a precaution, we attempted to delete the file, but it was not successful. \n To be sure the content is the same, you should try removing the file manually, \n then uploading the file again. \n \n\nThe delete attempt failed with this message:\n\n ";
FileExceptionMessages["UPLOAD_MISMATCH"] = "The uploaded data did not match the data from the server. \n As a precaution, the file has been deleted. \n To be sure the content is the same, you should try uploading the file again.";
})(FileExceptionMessages = exports.FileExceptionMessages || (exports.FileExceptionMessages = {}));
/**
* A File object is created from your {@link Bucket} object using
* {@link Bucket#file}.
*
* @class
*/
class File extends nodejs_common_1.ServiceObject {
/**
* Cloud Storage uses access control lists (ACLs) to manage object and
* bucket access. ACLs are the mechanism you use to share objects with other
* users and allow other users to access your buckets and objects.
*
* An ACL consists of one or more entries, where each entry grants permissions
* to an entity. Permissions define the actions that can be performed against
* an object or bucket (for example, `READ` or `WRITE`); the entity defines
* who the permission applies to (for example, a specific user or group of
* users).
*
* The `acl` object on a File instance provides methods to get you a list of
* the ACLs defined on your bucket, as well as set, update, and delete them.
*
* See {@link http://goo.gl/6qBBPO| About Access Control lists}
*
* @name File#acl
* @mixes Acl
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
* //-
* // Make a file publicly readable.
* //-
* const options = {
* entity: 'allUsers',
* role: storage.acl.READER_ROLE
* };
*
* file.acl.add(options, function(err, aclObject) {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.acl.add(options).then(function(data) {
* const aclObject = data[0];
* const apiResponse = data[1];
* });
* ```
*/
/**
* The API-formatted resource description of the file.
*
* Note: This is not guaranteed to be up-to-date when accessed. To get the
* latest record, call the `getMetadata()` method.
*
* @name File#metadata
* @type {object}
*/
/**
* The file's name.
* @name File#name
* @type {string}
*/
/**
* @typedef {object} FileOptions Options passed to the File constructor.
* @property {string} [encryptionKey] A custom encryption key.
* @property {number} [generation] Generation to scope the file to.
* @property {string} [kmsKeyName] Cloud KMS Key used to encrypt this
* object, if the object is encrypted by such a key. Limited availability;
* usable only by enabled projects.
* @property {string} [userProject] The ID of the project which will be
* billed for all requests made from File object.
*/
/**
* Constructs a file object.
*
* @param {Bucket} bucket The Bucket instance this file is
* attached to.
* @param {string} name The name of the remote file.
* @param {FileOptions} [options] Configuration options.
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
* ```
*/
constructor(bucket, name, options = {}) {
var _a, _b;
const requestQueryObject = {};
let generation;
if (options.generation !== null) {
if (typeof options.generation === 'string') {
generation = Number(options.generation);
}
else {
generation = options.generation;
}
if (!isNaN(generation)) {
requestQueryObject.generation = generation;
}
}
Object.assign(requestQueryObject, options.preconditionOpts);
const userProject = options.userProject || bucket.userProject;
if (typeof userProject === 'string') {
requestQueryObject.userProject = userProject;
}
const methods = {
/**
* @typedef {array} DeleteFileResponse
* @property {object} 0 The full API response.
*/
/**
* @callback DeleteFileCallback
* @param {?Error} err Request error, if any.
* @param {object} apiResponse The full API response.
*/
/**
* Delete the file.
*
* See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/delete| Objects: delete API Documentation}
*
* @method File#delete
* @param {object} [options] Configuration options.
* @param {boolean} [options.ignoreNotFound = false] Ignore an error if
* the file does not exist.
* @param {string} [options.userProject] The ID of the project which will be
* billed for the request.
* @param {DeleteFileCallback} [callback] Callback function.
* @returns {Promise<DeleteFileResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
* file.delete(function(err, apiResponse) {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.delete().then(function(data) {
* const apiResponse = data[0];
* });
*
* ```
* @example <caption>include:samples/files.js</caption>
* region_tag:storage_delete_file
* Another example:
*/
delete: {
reqOpts: {
qs: requestQueryObject,
},
},
/**
* @typedef {array} FileExistsResponse
* @property {boolean} 0 Whether the {@link File} exists.
*/
/**
* @callback FileExistsCallback
* @param {?Error} err Request error, if any.
* @param {boolean} exists Whether the {@link File} exists.
*/
/**
* Check if the file exists.
*
* @method File#exists
* @param {options} [options] Configuration options.
* @param {string} [options.userProject] The ID of the project which will be
* billed for the request.
* @param {FileExistsCallback} [callback] Callback function.
* @returns {Promise<FileExistsResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
*
* file.exists(function(err, exists) {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.exists().then(function(data) {
* const exists = data[0];
* });
* ```
*/
exists: {
reqOpts: {
qs: requestQueryObject,
},
},
/**
* @typedef {array} GetFileResponse
* @property {File} 0 The {@link File}.
* @property {object} 1 The full API response.
*/
/**
* @callback GetFileCallback
* @param {?Error} err Request error, if any.
* @param {File} file The {@link File}.
* @param {object} apiResponse The full API response.
*/
/**
* Get a file object and its metadata if it exists.
*
* @method File#get
* @param {options} [options] Configuration options.
* @param {string} [options.userProject] The ID of the project which will be
* billed for the request.
* @param {GetFileCallback} [callback] Callback function.
* @returns {Promise<GetFileResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
*
* file.get(function(err, file, apiResponse) {
* // file.metadata` has been populated.
* });
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.get().then(function(data) {
* const file = data[0];
* const apiResponse = data[1];
* });
* ```
*/
get: {
reqOpts: {
qs: requestQueryObject,
},
},
/**
* @typedef {array} GetFileMetadataResponse
* @property {object} 0 The {@link File} metadata.
* @property {object} 1 The full API response.
*/
/**
* @callback GetFileMetadataCallback
* @param {?Error} err Request error, if any.
* @param {object} metadata The {@link File} metadata.
* @param {object} apiResponse The full API response.
*/
/**
* Get the file's metadata.
*
* See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/get| Objects: get API Documentation}
*
* @method File#getMetadata
* @param {object} [options] Configuration options.
* @param {string} [options.userProject] The ID of the project which will be
* billed for the request.
* @param {GetFileMetadataCallback} [callback] Callback function.
* @returns {Promise<GetFileMetadataResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
*
* file.getMetadata(function(err, metadata, apiResponse) {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.getMetadata().then(function(data) {
* const metadata = data[0];
* const apiResponse = data[1];
* });
*
* ```
* @example <caption>include:samples/files.js</caption>
* region_tag:storage_get_metadata
* Another example:
*/
getMetadata: {
reqOpts: {
qs: requestQueryObject,
},
},
/**
* @typedef {object} SetFileMetadataOptions Configuration options for File#setMetadata().
* @param {string} [userProject] The ID of the project which will be billed for the request.
*/
/**
* @callback SetFileMetadataCallback
* @param {?Error} err Request error, if any.
* @param {object} apiResponse The full API response.
*/
/**
* @typedef {array} SetFileMetadataResponse
* @property {object} 0 The full API response.
*/
/**
* Merge the given metadata with the current remote file's metadata. This
* will set metadata if it was previously unset or update previously set
* metadata. To unset previously set metadata, set its value to null.
*
* You can set custom key/value pairs in the metadata key of the given
* object, however the other properties outside of this object must adhere
* to the {@link https://goo.gl/BOnnCK| official API documentation}.
*
*
* See the examples below for more information.
*
* See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/patch| Objects: patch API Documentation}
*
* @method File#setMetadata
* @param {object} [metadata] The metadata you wish to update.
* @param {SetFileMetadataOptions} [options] Configuration options.
* @param {SetFileMetadataCallback} [callback] Callback function.
* @returns {Promise<SetFileMetadataResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
*
* const metadata = {
* contentType: 'application/x-font-ttf',
* metadata: {
* my: 'custom',
* properties: 'go here'
* }
* };
*
* file.setMetadata(metadata, function(err, apiResponse) {});
*
* // Assuming current metadata = { hello: 'world', unsetMe: 'will do' }
* file.setMetadata({
* metadata: {
* abc: '123', // will be set.
* unsetMe: null, // will be unset (deleted).
* hello: 'goodbye' // will be updated from 'world' to 'goodbye'.
* }
* }, function(err, apiResponse) {
* // metadata should now be { abc: '123', hello: 'goodbye' }
* });
*
* //-
* // Set a temporary hold on this file from its bucket's retention period
* // configuration.
* //
* file.setMetadata({
* temporaryHold: true
* }, function(err, apiResponse) {});
*
* //-
* // Alternatively, you may set a temporary hold. This will follow the
* // same behavior as an event-based hold, with the exception that the
* // bucket's retention policy will not renew for this file from the time
* // the hold is released.
* //-
* file.setMetadata({
* eventBasedHold: true
* }, function(err, apiResponse) {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.setMetadata(metadata).then(function(data) {
* const apiResponse = data[0];
* });
* ```
*/
setMetadata: {
reqOpts: {
qs: requestQueryObject,
},
},
};
super({
parent: bucket,
baseUrl: '/o',
id: encodeURIComponent(name),
methods,
});
this.bucket = bucket;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.storage = bucket.parent;
// @TODO Can this duplicate code from above be avoided?
if (options.generation !== null) {
let generation;
if (typeof options.generation === 'string') {
generation = Number(options.generation);
}
else {
generation = options.generation;
}
if (!isNaN(generation)) {
this.generation = generation;
}
}
this.kmsKeyName = options.kmsKeyName;
this.userProject = userProject;
this.name = name;
if (options.encryptionKey) {
this.setEncryptionKey(options.encryptionKey);
}
this.acl = new acl_1.Acl({
request: this.request.bind(this),
pathPrefix: '/acl',
});
this.instanceRetryValue = (_b = (_a = this.storage) === null || _a === void 0 ? void 0 : _a.retryOptions) === null || _b === void 0 ? void 0 : _b.autoRetry;
this.instancePreconditionOpts = options === null || options === void 0 ? void 0 : options.preconditionOpts;
}
/**
* A helper method for determining if a request should be retried based on preconditions.
* This should only be used for methods where the idempotency is determined by
* `ifGenerationMatch`
* @private
*
* A request should not be retried under the following conditions:
* - if precondition option `ifGenerationMatch` is not set OR
* - if `idempotencyStrategy` is set to `RetryNever`
*/
shouldRetryBasedOnPreconditionAndIdempotencyStrat(options) {
var _a;
return !(((options === null || options === void 0 ? void 0 : options.ifGenerationMatch) === undefined &&
((_a = this.instancePreconditionOpts) === null || _a === void 0 ? void 0 : _a.ifGenerationMatch) === undefined &&
this.storage.retryOptions.idempotencyStrategy ===
storage_1.IdempotencyStrategy.RetryConditional) ||
this.storage.retryOptions.idempotencyStrategy ===
storage_1.IdempotencyStrategy.RetryNever);
}
/**
* @typedef {array} CopyResponse
* @property {File} 0 The copied {@link File}.
* @property {object} 1 The full API response.
*/
/**
* @callback CopyCallback
* @param {?Error} err Request error, if any.
* @param {File} copiedFile The copied {@link File}.
* @param {object} apiResponse The full API response.
*/
/**
* @typedef {object} CopyOptions Configuration options for File#copy(). See an
* {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}.
* @property {string} [cacheControl] The cacheControl setting for the new file.
* @property {string} [contentEncoding] The contentEncoding setting for the new file.
* @property {string} [contentType] The contentType setting for the new file.
* @property {string} [destinationKmsKeyName] Resource name of the Cloud
* KMS key, of the form
* `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`,
* that will be used to encrypt the object. Overwrites the object
* metadata's `kms_key_name` value, if any.
* @property {Metadata} [metadata] Metadata to specify on the copied file.
* @property {string} [predefinedAcl] Set the ACL for the new file.
* @property {string} [token] A previously-returned `rewriteToken` from an
* unfinished rewrite request.
* @property {string} [userProject] The ID of the project which will be
* billed for the request.
*/
/**
* Copy this file to another file. By default, this will copy the file to the
* same bucket, but you can choose to copy it to another Bucket by providing
* a Bucket or File object or a URL starting with "gs://".
* The generation of the file will not be preserved.
*
* See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite| Objects: rewrite API Documentation}
*
* @throws {Error} If the destination file is not provided.
*
* @param {string|Bucket|File} destination Destination file.
* @param {CopyOptions} [options] Configuration options. See an
* @param {CopyCallback} [callback] Callback function.
* @returns {Promise<CopyResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
*
* //-
* // You can pass in a variety of types for the destination.
* //
* // For all of the below examples, assume we are working with the following
* // Bucket and File objects.
* //-
* const bucket = storage.bucket('my-bucket');
* const file = bucket.file('my-image.png');
*
* //-
* // If you pass in a string for the destination, the file is copied to its
* // current bucket, under the new name provided.
* //-
* file.copy('my-image-copy.png', function(err, copiedFile, apiResponse) {
* // `my-bucket` now contains:
* // - "my-image.png"
* // - "my-image-copy.png"
*
* // `copiedFile` is an instance of a File object that refers to your new
* // file.
* });
*
* //-
* // If you pass in a string starting with "gs://" for the destination, the
* // file is copied to the other bucket and under the new name provided.
* //-
* const newLocation = 'gs://another-bucket/my-image-copy.png';
* file.copy(newLocation, function(err, copiedFile, apiResponse) {
* // `my-bucket` still contains:
* // - "my-image.png"
* //
* // `another-bucket` now contains:
* // - "my-image-copy.png"
*
* // `copiedFile` is an instance of a File object that refers to your new
* // file.
* });
*
* //-
* // If you pass in a Bucket object, the file will be copied to that bucket
* // using the same name.
* //-
* const anotherBucket = storage.bucket('another-bucket');
* file.copy(anotherBucket, function(err, copiedFile, apiResponse) {
* // `my-bucket` still contains:
* // - "my-image.png"
* //
* // `another-bucket` now contains:
* // - "my-image.png"
*
* // `copiedFile` is an instance of a File object that refers to your new
* // file.
* });
*
* //-
* // If you pass in a File object, you have complete control over the new
* // bucket and filename.
* //-
* const anotherFile = anotherBucket.file('my-awesome-image.png');
* file.copy(anotherFile, function(err, copiedFile, apiResponse) {
* // `my-bucket` still contains:
* // - "my-image.png"
* //
* // `another-bucket` now contains:
* // - "my-awesome-image.png"
*
* // Note:
* // The `copiedFile` parameter is equal to `anotherFile`.
* });
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.copy(newLocation).then(function(data) {
* const newFile = data[0];
* const apiResponse = data[1];
* });
*
* ```
* @example <caption>include:samples/files.js</caption>
* region_tag:storage_copy_file
* Another example:
*/
copy(destination, optionsOrCallback, callback) {
const noDestinationError = new Error(FileExceptionMessages.DESTINATION_NO_NAME);
if (!destination) {
throw noDestinationError;
}
let options = {};
if (typeof optionsOrCallback === 'function') {
callback = optionsOrCallback;
}
else if (optionsOrCallback) {
options = optionsOrCallback;
}
options = extend(true, {}, options);
callback = callback || nodejs_common_1.util.noop;
let destBucket;
let destName;
let newFile;
if (typeof destination === 'string') {
const parsedDestination = GS_URL_REGEXP.exec(destination);
if (parsedDestination !== null && parsedDestination.length === 3) {
destBucket = this.storage.bucket(parsedDestination[1]);
destName = parsedDestination[2];
}
else {
destBucket = this.bucket;
destName = destination;
}
}
else if (destination instanceof bucket_1.Bucket) {
destBucket = destination;
destName = this.name;
}
else if (destination instanceof File) {
destBucket = destination.bucket;
destName = destination.name;
newFile = destination;
}
else {
throw noDestinationError;
}
const query = {};
if (this.generation !== undefined) {
query.sourceGeneration = this.generation;
}
if (options.token !== undefined) {
query.rewriteToken = options.token;
}
if (options.userProject !== undefined) {
query.userProject = options.userProject;
delete options.userProject;
}
if (options.predefinedAcl !== undefined) {
query.destinationPredefinedAcl = options.predefinedAcl;
delete options.predefinedAcl;
}
newFile = newFile || destBucket.file(destName);
const headers = {};
if (this.encryptionKey !== undefined) {
headers['x-goog-copy-source-encryption-algorithm'] = 'AES256';
headers['x-goog-copy-source-encryption-key'] = this.encryptionKeyBase64;
headers['x-goog-copy-source-encryption-key-sha256'] =
this.encryptionKeyHash;
}
if (newFile.encryptionKey !== undefined) {
this.setEncryptionKey(newFile.encryptionKey);
}
else if (options.destinationKmsKeyName !== undefined) {
query.destinationKmsKeyName = options.destinationKmsKeyName;
delete options.destinationKmsKeyName;
}
else if (newFile.kmsKeyName !== undefined) {
query.destinationKmsKeyName = newFile.kmsKeyName;
}
if (query.destinationKmsKeyName) {
this.kmsKeyName = query.destinationKmsKeyName;
const keyIndex = this.interceptors.indexOf(this.encryptionKeyInterceptor);
if (keyIndex > -1) {
this.interceptors.splice(keyIndex, 1);
}
}
this.request({
method: 'POST',
uri: `/rewriteTo/b/${destBucket.name}/o/${encodeURIComponent(newFile.name)}`,
qs: query,
json: options,
headers,
}, (err, resp) => {
if (err) {
callback(err, null, resp);
return;
}
if (resp.rewriteToken) {
const options = {
token: resp.rewriteToken,
};
if (query.userProject) {
options.userProject = query.userProject;
}
if (query.destinationKmsKeyName) {
options.destinationKmsKeyName = query.destinationKmsKeyName;
}
this.copy(newFile, options, callback);
return;
}
callback(null, newFile, resp);
});
}
/**
* @typedef {object} CreateReadStreamOptions Configuration options for File#createReadStream.
* @property {string} [userProject] The ID of the project which will be
* billed for the request.
* @property {string|boolean} [validation] Possible values: `"md5"`,
* `"crc32c"`, or `false`. By default, data integrity is validated with a
* CRC32c checksum. You may use MD5 if preferred, but that hash is not
* supported for composite objects. An error will be raised if MD5 is
* specified but is not available. You may also choose to skip validation
* completely, however this is **not recommended**.
* @property {number} [start] A byte offset to begin the file's download
* from. Default is 0. NOTE: Byte ranges are inclusive; that is,
* `options.start = 0` and `options.end = 999` represent the first 1000
* bytes in a file or object. NOTE: when specifying a byte range, data
* integrity is not available.
* @property {number} [end] A byte offset to stop reading the file at.
* NOTE: Byte ranges are inclusive; that is, `options.start = 0` and
* `options.end = 999` represent the first 1000 bytes in a file or object.
* NOTE: when specifying a byte range, data integrity is not available.
* @property {boolean} [decompress=true] Disable auto decompression of the
* received data. By default this option is set to `true`.
* Applicable in cases where the data was uploaded with
* `gzip: true` option. See {@link File#createWriteStream}.
*/
/**
* Create a readable stream to read the contents of the remote file. It can be
* piped to a writable stream or listened to for 'data' events to read a
* file's contents.
*
* In the unlikely event there is a mismatch between what you downloaded and
* the version in your Bucket, your error handler will receive an error with
* code "CONTENT_DOWNLOAD_MISMATCH". If you receive this error, the best
* recourse is to try downloading the file again.
*
* For faster crc32c computation, you must manually install
* {@link https://www.npmjs.com/package/fast-crc32c| `fast-crc32c`}:
*
* $ npm install --save fast-crc32c
*
* NOTE: Readable streams will emit the `end` event when the file is fully
* downloaded.
*
* @param {CreateReadStreamOptions} [options] Configuration options.
* @returns {ReadableStream}
*
* @example
* ```
* //-
* // <h4>Downloading a File</h4>
* //
* // The example below demonstrates how we can reference a remote file, then
* // pipe its contents to a local file. This is effectively creating a local
* // backup of your remote data.
* //-
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const bucket = storage.bucket('my-bucket');
*
* const fs = require('fs');
* const remoteFile = bucket.file('image.png');
* const localFilename = '/Users/stephen/Photos/image.png';
*
* remoteFile.createReadStream()
* .on('error', function(err) {})
* .on('response', function(response) {
* // Server connected and responded with the specified status and headers.
* })
* .on('end', function() {
* // The file is fully downloaded.
* })
* .pipe(fs.createWriteStream(localFilename));
*
* //-
* // To limit the downloaded data to only a byte range, pass an options
* // object.
* //-
* const logFile = myBucket.file('access_log');
* logFile.createReadStream({
* start: 10000,
* end: 20000
* })
* .on('error', function(err) {})
* .pipe(fs.createWriteStream('/Users/stephen/logfile.txt'));
*
* //-
* // To read a tail byte range, specify only `options.end` as a negative
* // number.
* //-
* const logFile = myBucket.file('access_log');
* logFile.createReadStream({
* end: -100
* })
* .on('error', function(err) {})
* .pipe(fs.createWriteStream('/Users/stephen/logfile.txt'));
* ```
*/
createReadStream(options = {}) {
options = Object.assign({ decompress: true }, options);
const rangeRequest = typeof options.start === 'number' || typeof options.end === 'number';
const tailRequest = options.end < 0;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let validateStream; // Created later, if necessary.
const throughStream = streamEvents(new stream_1.PassThrough());
let isServedCompressed = true;
let crc32c = true;
let md5 = false;
if (typeof options.validation === 'string') {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
options.validation = options.validation.toLowerCase();
crc32c = options.validation === 'crc32c';
md5 = options.validation === 'md5';
}
else if (options.validation === false) {
crc32c = false;
}
const shouldRunValidation = !rangeRequest && (crc32c || md5);
if (rangeRequest) {
if (typeof options.validation === 'string' ||
options.validation === true) {
throw new Error(FileExceptionMessages.INVALID_VALIDATION_FILE_RANGE);
}
// Range requests can't receive data integrity checks.
crc32c = false;
md5 = false;
}
// Authenticate the request, then pipe the remote API request to the stream
// returned to the user.
const makeRequest = () => {
const query = {
alt: 'media',
};
if (this.generation) {
query.generation = this.generation;
}
if (options.userProject) {
query.userProject = options.userProject;
}
const headers = {
'Accept-Encoding': 'gzip',
'Cache-Control': 'no-store',
};
if (rangeRequest) {
const start = typeof options.start === 'number' ? options.start : '0';
const end = typeof options.end === 'number' ? options.end : '';
headers.Range = `bytes=${tailRequest ? end : `${start}-${end}`}`;
}
const reqOpts = {
forever: false,
uri: '',
headers,
qs: query,
};
const hashes = {};
this.requestStream(reqOpts)
.on('error', err => {
throughStream.destroy(err);
})
.on('response', res => {
throughStream.emit('response', res);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
nodejs_common_1.util.handleResp(null, res, null, onResponse);
})
.resume();
// We listen to the response event from the request stream so that we
// can...
//
// 1) Intercept any data from going to the user if an error occurred.
// 2) Calculate the hashes from the http.IncomingMessage response
// stream,
// which will return the bytes from the source without decompressing
// gzip'd content. We then send it through decompressed, if
// applicable, to the user.
const onResponse = (err, _body, rawResponseStream) => {
if (err) {
// Get error message from the body.
getStream(rawResponseStream).then(body => {
err.message = body;
throughStream.destroy(err);
});
return;
}
rawResponseStream.on('error', onComplete);
const headers = rawResponseStream.toJSON().headers;
isServedCompressed = headers['content-encoding'] === 'gzip';
const throughStreams = [];
if (shouldRunValidation) {
// The x-goog-hash header should be set with a crc32c and md5 hash.
// ex: headers['x-goog-hash'] = 'crc32c=xxxx,md5=xxxx'
if (typeof headers['x-goog-hash'] === 'string') {
headers['x-goog-hash']
.split(',')
.forEach((hashKeyValPair) => {
const delimiterIndex = hashKeyValPair.indexOf('=');
const hashType = hashKeyValPair.substr(0, delimiterIndex);
const hashValue = hashKeyValPair.substr(delimiterIndex + 1);
hashes[hashType] = hashValue;
});
}
validateStream = hashStreamValidation({ crc32c, md5 });
throughStreams.push(validateStream);
}
if (isServedCompressed && options.decompress) {
throughStreams.push(zlib.createGunzip());
}
if (throughStreams.length === 1) {
rawResponseStream =
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rawResponseStream.pipe(throughStreams[0]);
}
else if (throughStreams.length > 1) {
rawResponseStream = rawResponseStream.pipe(pumpify.obj(throughStreams));
}
rawResponseStream
.on('error', onComplete)
.on('end', onComplete)
.pipe(throughStream, { end: false });
};
// This is hooked to the `complete` event from the request stream. This is
// our chance to validate the data and let the user know if anything went
// wrong.
let onCompleteCalled = false;
const onComplete = async (err) => {
if (onCompleteCalled) {
return;
}
onCompleteCalled = true;
if (err) {
throughStream.destroy(err);
return;
}
if (rangeRequest || !shouldRunValidation) {
throughStream.end();
return;
}
// TODO(https://github.com/googleapis/nodejs-storage/issues/709):
// Remove once the backend issue is fixed.
// If object is stored compressed (having
// metadata.contentEncoding === 'gzip') and was served decompressed,
// then skip checksum validation because the remote checksum is computed
// against the compressed version of the object.
if (!isServedCompressed) {
try {
await this.getMetadata({ userProject: options.userProject });
}
catch (e) {
throughStream.destroy(e);
return;
}
if (this.metadata.contentEncoding === 'gzip') {
throughStream.end();
return;
}
}
// If we're doing validation, assume the worst-- a data integrity
// mismatch. If not, these tests won't be performed, and we can assume
// the best.
let failed = crc32c || md5;
if (crc32c && hashes.crc32c) {
// We must remove the first four bytes from the returned checksum.
// http://stackoverflow.com/questions/25096737/
// base64-encoding-of-crc32c-long-value
failed = !validateStream.test('crc32c', hashes.crc32c.substr(4));
}
if (md5 && hashes.md5) {
failed = !validateStream.test('md5', hashes.md5);
}
if (md5 && !hashes.md5) {
const hashError = new RequestError(FileExceptionMessages.MD5_NOT_AVAILABLE);
hashError.code = 'MD5_NOT_AVAILABLE';
throughStream.destroy(hashError);
}
else if (failed) {
const mismatchError = new RequestError(FileExceptionMessages.DOWNLOAD_MISMATCH);
mismatchError.code = 'CONTENT_DOWNLOAD_MISMATCH';
throughStream.destroy(mismatchError);
}
else {
throughStream.end();
}
};
};
throughStream.on('reading', makeRequest);
return throughStream;
}
/**
* @callback CreateResumableUploadCallback
* @param {?Error} err Request error, if any.
* @param {string} uri The resumable upload's unique session URI.
*/
/**
* @typedef {array} CreateResumableUploadResponse
* @property {string} 0 The resumable upload's unique session URI.
*/
/**
* @typedef {object} CreateResumableUploadOptions
* @property {string} [configPath] A full JSON file path to use with
* `gcs-resumable-upload`. This maps to the {@link https://github.com/yeoman/configstore/tree/0df1ec950d952b1f0dfb39ce22af8e505dffc71a#configpath| configstore option by the same name}.
* @property {object} [metadata] Metadata to set on the file.
* @property {number} [offset] The starting byte of the upload stream for resuming an interrupted upload.
* @property {string} [origin] Origin header to set for the upload.
* @property {string} [predefinedAcl] Apply a predefined set of access
* controls to this object.
*
* Acceptable values are:
* - **`authenticatedRead`** - Object owner gets `OWNER` access, and
* `allAuthenticatedUsers` get `READER` access.
*
* - **`bucketOwnerFullControl`** - Object owner gets `OWNER` access, and
* project team owners get `OWNER` access.
*
* - **`bucketOwnerRead`** - Object owner gets `OWNER` access, and project
* team owners get `READER` access.
*
* - **`private`** - Object owner gets `OWNER` access.
*
* - **`projectPrivate`** - Object owner gets `OWNER` access, and project
* team members get access according to their roles.
*
* - **`publicRead`** - Object owner gets `OWNER` access, and `allUsers`
* get `READER` access.
* @property {boolean} [private] Make the uploaded file private. (Alias for
* `options.predefinedAcl = 'private'`)
* @property {boolean} [public] Make the uploaded file public. (Alias for
* `options.predefinedAcl = 'publicRead'`)
* @property {string} [userProject] The ID of the project which will be
* billed for the request.
* @property {string} [chunkSize] Create a separate request per chunk. Should
* be a multiple of 256 KiB (2^18).
* {@link https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload| We recommend using at least 8 MiB for the chunk size.}
*/
/**
* Create a unique resumable upload session URI. This is the first step when
* performing a resumable upload.
*
* See the {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload| Resumable upload guide}
* for more on how the entire process works.
*
* <h4>Note</h4>
*
* If you are just looking to perform a resumable upload without worrying
* about any of the details, see {@link File#createWriteStream}. Resumable
* uploads are performed by default.
*
* See {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload| Resumable upload guide}
*
* @param {CreateResumableUploadOptions} [options] Configuration options.
* @param {CreateResumableUploadCallback} [callback] Callback function.
* @returns {Promise<CreateResumableUploadResponse>}
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file');
* file.createResumableUpload(function(err, uri) {
* if (!err) {
* // `uri` can be used to PUT data to.
* }
* });
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* file.createResumableUpload().then(function(data) {
* const uri = data[0];
* });
* ```
*/
createResumableUpload(optionsOrCallback, callback) {
var _a, _b;
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
callback =
typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
const retryOptions = this.storage.retryOptions;
if ((((_a = options === null || options === void 0 ? void 0 : options.preconditionOpts) === null || _a === void 0 ? void 0 : _a.ifGenerationMatch) === undefined &&
((_b = this.instancePreconditionOpts) === null || _b === void 0 ? void 0 : _b.ifGenerationMatch) === undefined &&
this.storage.retryOptions.idempotencyStrategy ===
storage_1.IdempotencyStrategy.RetryConditional) ||
this.storage.retryOptions.idempotencyStrategy ===
storage_1.IdempotencyStrategy.RetryNever) {
retryOptions.autoRetry = false;
}
resumableUpload.createURI({
authClient: this.storage.authClient,
apiEndpoint: this.storage.apiEndpoint,
bucket: this.bucket.name,
configPath: options.configPath,
customRequestOptions: this.getRequestInterceptors().reduce((reqOpts, interceptorFn) => interceptorFn(reqOpts), {}),
file: this.name,
generation: this.generation,
key: this.encryptionKey,
kmsKeyName: this.kmsKeyName,
metadata: options.metadata,