@google-cloud/bigquery
Version:
Google BigQuery Client Library for Node.js
1,238 lines • 46.5 kB
JavaScript
"use strict";
/*!
* Copyright 2014 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.Table = void 0;
const common_1 = require("@google-cloud/common");
const paginator_1 = require("@google-cloud/paginator");
const promisify_1 = require("@google-cloud/promisify");
const arrify = require("arrify");
const Big = require("big.js");
const extend = require("extend");
const events_1 = require("events");
const fs = require("fs");
const is = require("is");
const path = require("path");
const streamEvents = require("stream-events");
const uuid = require("uuid");
const _1 = require(".");
const stream_1 = require("stream");
const rowQueue_1 = require("./rowQueue");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const duplexify = require('duplexify');
/**
* The file formats accepted by BigQuery.
*
* @type {object}
* @private
*/
const FORMATS = {
avro: 'AVRO',
csv: 'CSV',
export_metadata: 'DATASTORE_BACKUP',
json: 'NEWLINE_DELIMITED_JSON',
orc: 'ORC',
parquet: 'PARQUET',
};
/**
* Table objects are returned by methods such as
* {@link Dataset#table}, {@link Dataset#createTable}, and
* {@link Dataset#getTables}.
*
* @class
* @param {Dataset} dataset {@link Dataset} instance.
* @param {string} id The ID of the table.
* @param {object} [options] Table options.
* @param {string} [options.location] The geographic location of the table, by
* default this value is inherited from the dataset. This can be used to
* configure the location of all jobs created through a table instance. It
* cannot be used to set the actual location of the table. This value will
* be superseded by any API responses containing location data for the
* table.
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
* ```
*/
class Table extends common_1.ServiceObject {
createReadStream(options) {
// placeholder body, overwritten in constructor
return new paginator_1.ResourceStream({}, () => { });
}
constructor(dataset, id, options) {
const methods = {
/**
* @callback CreateTableCallback
* @param {?Error} err Request error, if any.
* @param {Table} table The table.
* @param {object} apiResponse The full API response body.
*/
/**
* @typedef {array} CreateTableResponse
* @property {Table} 0 The table.
* @property {object} 1 The full API response body.
*/
/**
* Create a table.
*
* @method Table#create
* @param {object} [options] See {@link Dataset#createTable}.
* @param {CreateTableCallback} [callback]
* @param {?error} callback.err An error returned while making this
* request.
* @param {Table} callback.table The new {@link Table}.
* @param {object} callback.apiResponse The full API response.
* @returns {Promise<CreateTableResponse>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
*
* table.create((err, table, apiResponse) => {
* if (!err) {
* // The table was created successfully.
* }
* });
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.create().then((data) => {
* const table = data[0];
* const apiResponse = data[1];
* });
* ```
*/
create: true,
/**
* @callback DeleteTableCallback
* @param {?Error} err Request error, if any.
* @param {object} apiResponse The full API response.
*/
/**
* @typedef {array} DeleteTableResponse
* @property {object} 0 The full API response.
*/
/**
* Delete a table and all its data.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/v2/tables/delete| Tables: delete API Documentation}
*
* @method Table#delete
* @param {DeleteTableCallback} [callback]
* @param {?error} callback.err An error returned while making this
* request.
* @param {object} callback.apiResponse The full API response.
* @returns {Promise<DeleteTableResponse>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
*
* table.delete((err, apiResponse) => {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.delete().then((data) => {
* const apiResponse = data[0];
* });
* ```
*/
delete: true,
/**
* @callback TableExistsCallback
* @param {?Error} err Request error, if any.
* @param {boolean} exists Indicates if the table exists.
*/
/**
* @typedef {array} TableExistsCallback
* @property {boolean} 0 Indicates if the table exists.
*/
/**
* Check if the table exists.
*
* @method Table#exists
* @param {TableExistsCallback} [callback]
* @param {?error} callback.err An error returned while making this
* request.
* @param {boolean} callback.exists Whether the table exists or not.
* @returns {Promise<TableExistsCallback>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
*
* table.exists((err, exists) => {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.exists().then((data) => {
* const exists = data[0];
* });
* ```
*/
exists: true,
/**
* @callback GetTableCallback
* @param {?Error} err Request error, if any.
* @param {Table} table The table.
* @param {object} apiResponse The full API response body.
*/
/**
* @typedef {array} GetTableResponse
* @property {Table} 0 The table.
* @property {object} 1 The full API response body.
*/
/**
* Get a table if it exists.
*
* You may optionally use this to "get or create" an object by providing
* an object with `autoCreate` set to `true`. Any extra configuration that
* is normally required for the `create` method must be contained within
* this object as well.
*
* If you wish to get a selection of metadata instead of the full table metadata
* (retrieved by both Table#get by default and by Table#getMetadata), use
* the `options` parameter to set the `view` and/or `selectedFields` query parameters.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get#TableMetadataView| Tables.get and TableMetadataView }
*
* @method Table#get
* @param {options} [options] Configuration object.
* @param {boolean} [options.autoCreate=false] Automatically create the
* object if it does not exist.
* @param {function} [callback]
* @param {?error} callback.err An error returned while making this
* request.
* @param {Table} callback.table The {@link Table}.
* @param {object} callback.apiResponse The full API response.
* @returns {Promise<GetTableResponse>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
*
* const options = {
* view: "BASIC"
* }
*
* table.get((err, table, apiResponse) => {
* // `table.metadata` has been populated.
* });
*
* table.get(options, (err, table, apiResponse) => {
* // A selection of `table.metadata` has been populated
* })
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.get().then((data) => {
* const table = data[0];
* const apiResponse = data[1];
* });
* ```
*/
get: true,
/**
* @callback GetTableMetadataCallback
* @param {?Error} err Request error, if any.
* @param {object} metadata The table metadata.
* @param {object} apiResponse The full API response.
*/
/**
* @typedef {array} GetTableMetadataResponse
* @property {object} 0 The table metadata.
* @property {object} 1 The full API response.
*/
/**
* Return the metadata associated with the Table.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/v2/tables/get| Tables: get API Documentation}
*
* @method Table#getMetadata
* @param {GetTableMetadataCallback} [callback] The callback function.
* @param {?error} callback.err An error returned while making this
* request.
* @param {object} callback.metadata The metadata of the Table.
* @param {object} callback.apiResponse The full API response.
* @returns {Promise<GetTableMetadataResponse>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
*
* const table = dataset.table('my-table');
*
* table.getMetadata((err, metadata, apiResponse) => {});
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.getMetadata().then((data) => {
* const metadata = data[0];
* const apiResponse = data[1];
* });
* ```
*/
getMetadata: true,
};
super({
parent: dataset,
baseUrl: '/tables',
id,
createMethod: dataset.createTable.bind(dataset),
methods,
});
if (options && options.location) {
this.location = options.location;
}
this.bigQuery = dataset.bigQuery;
this.dataset = dataset;
// Catch all for read-modify-write cycle
// https://cloud.google.com/bigquery/docs/api-performance#read-patch-write
this.interceptors.push({
request: (reqOpts) => {
if (reqOpts.method === 'PATCH' && reqOpts.json.etag) {
reqOpts.headers = reqOpts.headers || {};
reqOpts.headers['If-Match'] = reqOpts.json.etag;
}
return reqOpts;
},
});
/**
* Create a readable stream of the rows of data in your table. This method
* is simply a wrapper around {@link Table#getRows}.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list| Tabledata: list API Documentation}
*
* @returns {ReadableStream}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
* const table = dataset.table('my-table');
*
* table.createReadStream(options)
* .on('error', console.error)
* .on('data', row => {})
* .on('end', function() {
* // All rows have been retrieved.
* });
*
* //-
* // If you anticipate many results, you can end a stream early to prevent
* // unnecessary processing and API requests.
* //-
* table.createReadStream()
* .on('data', function(row) {
* this.end();
* });
* ```
*/
this.createReadStream = paginator_1.paginator.streamify('getRows');
}
/**
* Convert a comma-separated name:type string to a table schema object.
*
* @static
* @private
*
* @param {string} str Comma-separated schema string.
* @returns {object} Table schema in the format the API expects.
*/
static createSchemaFromString_(str) {
return str.split(/\s*,\s*/).reduce((acc, pair) => {
acc.fields.push({
name: pair.split(':')[0].trim(),
type: (pair.split(':')[1] || 'STRING').toUpperCase().trim(),
});
return acc;
}, {
fields: [],
});
}
/**
* Convert a row entry from native types to their encoded types that the API
* expects.
*
* @static
* @private
*
* @param {*} value The value to be converted.
* @returns {*} The converted value.
*/
static encodeValue_(value) {
var _a;
if (typeof value === 'undefined' || value === null) {
return null;
}
if (value instanceof Buffer) {
return value.toString('base64');
}
if (value instanceof Big) {
return value.toFixed();
}
const customTypeConstructorNames = [
'BigQueryDate',
'BigQueryDatetime',
'BigQueryInt',
'BigQueryTime',
'BigQueryTimestamp',
'Geography',
];
const constructorName = (_a = value.constructor) === null || _a === void 0 ? void 0 : _a.name;
const isCustomType = customTypeConstructorNames.indexOf(constructorName) > -1;
if (isCustomType) {
return value.value;
}
if (is.date(value)) {
return value.toJSON();
}
if (is.array(value)) {
return value.map(Table.encodeValue_);
}
if (typeof value === 'object') {
return Object.keys(value).reduce((acc, key) => {
acc[key] = Table.encodeValue_(value[key]);
return acc;
}, {});
}
return value;
}
/**
* @private
*/
static formatMetadata_(options) {
const body = extend(true, {}, options);
if (options.name) {
body.friendlyName = options.name;
delete body.name;
}
if (is.string(options.schema)) {
body.schema = Table.createSchemaFromString_(options.schema);
}
if (is.array(options.schema)) {
body.schema = {
fields: options.schema,
};
}
if (body.schema && body.schema.fields) {
body.schema.fields = body.schema.fields.map(field => {
if (field.fields) {
field.type = 'RECORD';
}
return field;
});
}
if (is.string(options.partitioning)) {
body.timePartitioning = {
type: options.partitioning.toUpperCase(),
};
delete body.partitioning;
}
if (is.string(options.view)) {
body.view = {
query: options.view,
useLegacySql: false,
};
}
return body;
}
copy(destination, metadataOrCallback, cb) {
const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
this.createCopyJob(destination, metadata, (err, job, resp) => {
if (err) {
callback(err, resp);
return;
}
job.on('error', callback).on('complete', (metadata) => {
callback(null, metadata);
});
});
}
copyFrom(sourceTables, metadataOrCallback, cb) {
const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
this.createCopyFromJob(sourceTables, metadata, (err, job, resp) => {
if (err) {
callback(err, resp);
return;
}
job.on('error', callback).on('complete', metadata => {
callback(null, metadata);
});
});
}
createCopyJob(destination, metadataOrCallback, cb) {
if (!(destination instanceof Table)) {
throw new Error('Destination must be a Table object.');
}
const metadata = typeof metadataOrCallback === 'object'
? metadataOrCallback
: {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const body = {
configuration: {
copy: extend(true, metadata, {
destinationTable: {
datasetId: destination.dataset.id,
projectId: destination.bigQuery.projectId,
tableId: destination.id,
},
sourceTable: {
datasetId: this.dataset.id,
projectId: this.bigQuery.projectId,
tableId: this.id,
},
}),
},
};
if (metadata.jobPrefix) {
body.jobPrefix = metadata.jobPrefix;
delete metadata.jobPrefix;
}
if (this.location) {
body.location = this.location;
}
if (metadata.jobId) {
body.jobId = metadata.jobId;
delete metadata.jobId;
}
this.bigQuery.createJob(body, callback);
}
createCopyFromJob(source, metadataOrCallback, cb) {
const sourceTables = arrify(source);
sourceTables.forEach(sourceTable => {
if (!(sourceTable instanceof Table)) {
throw new Error('Source must be a Table object.');
}
});
const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const body = {
configuration: {
copy: extend(true, metadata, {
destinationTable: {
datasetId: this.dataset.id,
projectId: this.bigQuery.projectId,
tableId: this.id,
},
sourceTables: sourceTables.map(sourceTable => {
return {
datasetId: sourceTable.dataset.id,
projectId: sourceTable.bigQuery.projectId,
tableId: sourceTable.id,
};
}),
}),
},
};
if (metadata.jobPrefix) {
body.jobPrefix = metadata.jobPrefix;
delete metadata.jobPrefix;
}
if (this.location) {
body.location = this.location;
}
if (metadata.jobId) {
body.jobId = metadata.jobId;
delete metadata.jobId;
}
this.bigQuery.createJob(body, callback);
}
createExtractJob(destination, optionsOrCallback, cb) {
let options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
options = extend(true, options, {
destinationUris: arrify(destination).map(dest => {
if (!common_1.util.isCustomType(dest, 'storage/file')) {
throw new Error('Destination must be a File object.');
}
// If no explicit format was provided, attempt to find a match from the
// file's extension. If no match, don't set, and default upstream to
// CSV.
const format = path.extname(dest.name).substr(1).toLowerCase();
if (!options.destinationFormat && !options.format && FORMATS[format]) {
options.destinationFormat = FORMATS[format];
}
return 'gs://' + dest.bucket.name + '/' + dest.name;
}),
});
if (options.format) {
options.format = options.format.toLowerCase();
if (FORMATS[options.format]) {
options.destinationFormat = FORMATS[options.format];
delete options.format;
}
else {
throw new Error('Destination format not recognized: ' + options.format);
}
}
if (options.gzip) {
options.compression = 'GZIP';
delete options.gzip;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const body = {
configuration: {
extract: extend(true, options, {
sourceTable: {
datasetId: this.dataset.id,
projectId: this.bigQuery.projectId,
tableId: this.id,
},
}),
},
};
if (options.jobPrefix) {
body.jobPrefix = options.jobPrefix;
delete options.jobPrefix;
}
if (this.location) {
body.location = this.location;
}
if (options.jobId) {
body.jobId = options.jobId;
delete options.jobId;
}
this.bigQuery.createJob(body, callback);
}
createLoadJob(source, metadataOrCallback, cb) {
const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
this._createLoadJob(source, metadata).then(([resp]) => callback(null, resp, resp.metadata), err => callback(err));
}
/**
* @param {string | File | File[]} source
* @param {JobLoadMetadata} metadata
* @returns {Promise<JobResponse>}
* @private
*/
async _createLoadJob(source, metadata) {
if (metadata.format) {
metadata.sourceFormat = FORMATS[metadata.format.toLowerCase()];
delete metadata.format;
}
if (this.location) {
metadata.location = this.location;
}
if (typeof source === 'string') {
// A path to a file was given. If a sourceFormat wasn't specified, try to
// find a match from the file's extension.
const detectedFormat = FORMATS[path.extname(source).substr(1).toLowerCase()];
if (!metadata.sourceFormat && detectedFormat) {
metadata.sourceFormat = detectedFormat;
}
// Read the file into a new write stream.
const jobWritable = fs
.createReadStream(source)
.pipe(this.createWriteStream_(metadata));
const [jobResponse] = (await (0, events_1.once)(jobWritable, 'job'));
return [jobResponse, jobResponse.metadata];
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const body = {
configuration: {
load: {
destinationTable: {
projectId: this.bigQuery.projectId,
datasetId: this.dataset.id,
tableId: this.id,
},
},
},
};
if (metadata.jobPrefix) {
body.jobPrefix = metadata.jobPrefix;
delete metadata.jobPrefix;
}
if (metadata.location) {
body.location = metadata.location;
delete metadata.location;
}
if (metadata.jobId) {
body.jobId = metadata.jobId;
delete metadata.jobId;
}
extend(true, body.configuration.load, metadata, {
sourceUris: arrify(source).map(src => {
if (!common_1.util.isCustomType(src, 'storage/file')) {
throw new Error('Source must be a File object.');
}
// If no explicit format was provided, attempt to find a match from
// the file's extension. If no match, don't set, and default upstream
// to CSV.
const format = FORMATS[path.extname(src.name).substr(1).toLowerCase()];
if (!metadata.sourceFormat && format) {
body.configuration.load.sourceFormat = format;
}
return 'gs://' + src.bucket.name + '/' + src.name;
}),
});
return this.bigQuery.createJob(body);
}
createQueryJob(options, callback) {
return this.dataset.createQueryJob(options, callback);
}
/**
* Run a query scoped to your dataset as a readable object stream.
*
* See {@link BigQuery#createQueryStream} for full documentation of this
* method.
*
* @param {object} query See {@link BigQuery#createQueryStream} for full
* documentation of this method.
* @returns {stream} See {@link BigQuery#createQueryStream} for full
* documentation of this method.
*/
createQueryStream(query) {
return this.dataset.createQueryStream(query);
}
/**
* Creates a write stream. Unlike the public version, this will not
* automatically poll the underlying job.
*
* @private
*
* @param {string|object} [metadata] Metadata to set with the load operation.
* The metadata object should be in the format of the
* {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad| `configuration.load`}
* property of a Jobs resource. If a string is given, it will be used
* as the filetype.
* @param {string} [metadata.jobId] Custom job id.
* @param {string} [metadata.jobPrefix] Prefix to apply to the job id.
* @returns {WritableStream}
*/
createWriteStream_(metadata) {
metadata = metadata || {};
if (typeof metadata === 'string') {
metadata = {
sourceFormat: FORMATS[metadata.toLowerCase()],
};
}
if (typeof metadata.schema === 'string') {
metadata.schema = Table.createSchemaFromString_(metadata.schema);
}
metadata = extend(true, {
destinationTable: {
projectId: this.bigQuery.projectId,
datasetId: this.dataset.id,
tableId: this.id,
},
}, metadata);
let jobId = metadata.jobId || uuid.v4();
if (metadata.jobId) {
delete metadata.jobId;
}
if (metadata.jobPrefix) {
jobId = metadata.jobPrefix + jobId;
delete metadata.jobPrefix;
}
const dup = streamEvents(duplexify());
dup.once('writing', () => {
common_1.util.makeWritableStream(dup, {
makeAuthenticatedRequest: this.bigQuery.makeAuthenticatedRequest,
metadata: {
configuration: {
load: metadata,
},
jobReference: {
jobId,
projectId: this.bigQuery.projectId,
location: this.location,
},
},
request: {
uri: `${this.bigQuery.apiEndpoint}/upload/bigquery/v2/projects/${this.bigQuery.projectId}/jobs`,
},
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(data) => {
const job = this.bigQuery.job(data.jobReference.jobId, {
location: data.jobReference.location,
});
job.metadata = data;
dup.emit('job', job);
});
});
return dup;
}
/**
* Load data into your table from a readable stream of AVRO, CSV, JSON, ORC,
* or PARQUET data.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/v2/jobs/insert| Jobs: insert API Documentation}
*
* @param {string|object} [metadata] Metadata to set with the load operation.
* The metadata object should be in the format of the
* {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad| `configuration.load`}
* property of a Jobs resource. If a string is given,
* it will be used as the filetype.
* @param {string} [metadata.jobId] Custom job id.
* @param {string} [metadata.jobPrefix] Prefix to apply to the job id.
* @returns {WritableStream}
*
* @throws {Error} If source format isn't recognized.
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
* const table = dataset.table('my-table');
*
* //-
* // Load data from a CSV file.
* //-
* const request = require('request');
*
* const csvUrl = 'http://goo.gl/kSE7z6';
*
* const metadata = {
* allowJaggedRows: true,
* skipLeadingRows: 1
* };
*
* request.get(csvUrl)
* .pipe(table.createWriteStream(metadata))
* .on('job', (job) => {
* // `job` is a Job object that can be used to check the status of the
* // request.
* })
* .on('complete', (job) => {
* // The job has completed successfully.
* });
*
* //-
* // Load data from a JSON file.
* //-
* const fs = require('fs');
*
* fs.createReadStream('./test/testdata/testfile.json')
* .pipe(table.createWriteStream('json'))
* .on('job', (job) => {
* // `job` is a Job object that can be used to check the status of the
* // request.
* })
* .on('complete', (job) => {
* // The job has completed successfully.
* });
* ```
*/
createWriteStream(metadata) {
const stream = this.createWriteStream_(metadata);
stream.on('prefinish', () => {
stream.cork();
});
stream.on('job', (job) => {
job
.on('error', err => {
stream.destroy(err);
})
.on('complete', () => {
stream.emit('complete', job);
stream.uncork();
});
});
return stream;
}
extract(destination, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
this.createExtractJob(destination, options, (err, job, resp) => {
if (err) {
callback(err, resp);
return;
}
job.on('error', callback).on('complete', metadata => {
callback(null, metadata);
});
});
}
/**
* Retrieves table data from a specified set of rows. The rows are returned to
* your callback as an array of objects matching your table's schema.
*
* See {@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list| Tabledata: list API Documentation}
*
* @param {object} [options] The configuration object.
* @param {boolean} [options.autoPaginate=true] Have pagination handled
* automatically.
* @param {number} [options.maxApiCalls] Maximum number of API calls to make.
* @param {number} [options.maxResults] Maximum number of results to return.
* @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values
* of 'INT64' type in {@link BigQueryInt} objects.
* If a `boolean`, this will wrap values in {@link BigQueryInt} objects.
* If an `object`, this will return a value returned by
* `wrapIntegers.integerTypeCastFunction`.
* @param {RowsCallback} [callback] The callback function. If `autoPaginate`
* is set to false a {@link ManualQueryResultsCallback} should be used.
* @param {?error} callback.err An error returned while making this request
* @param {array} callback.rows The table data from specified set of rows.
* @returns {Promise<RowsResponse>}
*
* @example
* ```
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('my-dataset');
* const table = dataset.table('my-table');
*
* table.getRows((err, rows) => {
* if (!err) {
* // rows is an array of results.
* }
* });
*
* //-
* // To control how many API requests are made and page through the results
* // manually, set `autoPaginate` to `false`.
* //-
* function manualPaginationCallback(err, rows, nextQuery, apiResponse) {
* if (nextQuery) {
* // More results exist.
* table.getRows(nextQuery, manualPaginationCallback);
* }
* }
*
* table.getRows({
* autoPaginate: false
* }, manualPaginationCallback);
*
* //-
* // If the callback is omitted, we'll return a Promise.
* //-
* table.getRows().then((data) => {
* const rows = data[0];
* });
* ```
*/
getRows(optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
const wrapIntegers = options.wrapIntegers ? options.wrapIntegers : false;
delete options.wrapIntegers;
const parseJSON = options.parseJSON ? options.parseJSON : false;
delete options.parseJSON;
const onComplete = (err, rows, nextQuery, resp) => {
if (err) {
callback(err, null, null, resp);
return;
}
rows = _1.BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], {
wrapIntegers: wrapIntegers,
selectedFields: options.selectedFields
? options.selectedFields.split(',')
: [],
parseJSON,
});
callback(null, rows, nextQuery, resp);
};
this.request({
uri: '/data',
qs: options,
}, (err, resp) => {
if (err) {
onComplete(err, null, null, resp);
return;
}
let nextQuery = null;
if (resp.pageToken) {
nextQuery = Object.assign({}, options, {
pageToken: resp.pageToken,
});
}
if (resp.rows && resp.rows.length > 0 && !this.metadata.schema) {
// We don't know the schema for this table yet. Do a quick stat.
this.getMetadata((err, metadata, apiResponse) => {
if (err) {
onComplete(err, null, null, apiResponse);
return;
}
onComplete(null, resp.rows, nextQuery, resp);
});
return;
}
onComplete(null, resp.rows, nextQuery, resp);
});
}
insert(rows, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object'
? optionsOrCallback
: {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
const promise = this._insertAndCreateTable(rows, options);
if (callback) {
promise.then(resp => callback(null, resp), err => callback(err, null));
}
else {
return promise.then(r => [r]);
}
}
/**
* Insert rows with retries, but will create the table if not exists.
*
* @param {RowMetadata | RowMetadata[]} rows
* @param {InsertRowsOptions} options
* @returns {Promise<bigquery.ITableDataInsertAllResponse | bigquery.ITable>}
* @private
*/
async _insertAndCreateTable(rows, options) {
const { schema } = options;
const delay = 60000;
try {
return await this._insertWithRetry(rows, options);
}
catch (err) {
if (err.code !== 404 || !schema) {
throw err;
}
}
try {
await this.create({ schema });
}
catch (err) {
if (err.code !== 409) {
throw err;
}
}
// table creation after failed access is subject to failure caching and
// eventual consistency, see:
// https://github.com/googleapis/google-cloud-python/issues/4553#issuecomment-350110292
await new Promise(resolve => setTimeout(resolve, delay));
return this._insertAndCreateTable(rows, options);
}
/**
* This method will attempt to insert rows while retrying any partial failures
* that occur along the way. Because partial insert failures are returned
* differently, we can't depend on our usual retry strategy.
*
* @private
*
* @param {RowMetadata|RowMetadata[]} rows The rows to insert.
* @param {InsertRowsOptions} options Insert options.
* @returns {Promise<bigquery.ITableDataInsertAllResponse>}
*/
async _insertWithRetry(rows, options) {
const { partialRetries = 3 } = options;
let error;
const maxAttempts = Math.max(partialRetries, 0) + 1;
for (let attempts = 0; attempts < maxAttempts; attempts++) {
try {
return await this._insert(rows, options);
}
catch (e) {
error = e;
rows = (e.errors || [])
.filter(err => !!err.row)
.map(err => err.row);
if (!rows.length) {
break;
}
}
}
throw error;
}
/**
* This method does the bulk of the work for processing options and making the
* network request.
*
* @private
*
* @param {RowMetadata|RowMetadata[]} rows The rows to insert.
* @param {InsertRowsOptions} options Insert options.
* @returns {Promise<bigquery.ITableDataInsertAllResponse>}
*/
async _insert(rows, options) {
rows = arrify(rows);
if (!rows.length) {
throw new Error('You must provide at least 1 row to be inserted.');
}
const json = extend(true, {}, options, { rows });
if (!options.raw) {
json.rows = rows.map((row) => {
const encoded = {
json: Table.encodeValue_(row),
};
if (options.createInsertId !== false) {
encoded.insertId = uuid.v4();
}
return encoded;
});
}
delete json.createInsertId;
delete json.partialRetries;
delete json.raw;
delete json.schema;
const [resp] = await this.request({
method: 'POST',
uri: '/insertAll',
json,
});
const partialFailures = (resp.insertErrors || []).map((insertError) => {
return {
errors: insertError.errors.map(error => {
return {
message: error.message,
reason: error.reason,
};
}),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
row: rows[insertError.index],
};
});
if (partialFailures.length > 0) {
throw new common_1.util.PartialFailureError({
errors: partialFailures,
response: resp,
});
}
return resp;
}
createInsertStream(options) {
options = typeof options === 'object' ? options : {};
const dup = new stream_1.Duplex({ objectMode: true });
dup._write = (chunk, encoding, cb) => {
this.rowQueue.add(chunk, () => { });
cb();
};
this.rowQueue = new rowQueue_1.RowQueue(this, dup, options);
return dup;
}
load(source, metadataOrCallback, cb) {
const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
this.createLoadJob(source, metadata, (err, job, resp) => {
if (err) {
callback(err, resp);
return;
}
job.on('error', callback).on('complete', metadata => {
callback(null, metadata);
});
});
}
query(query, callback) {
if (typeof query === 'string') {
query = {
query,
};
}
this.dataset.query(query, callback);
}
setMetadata(metadata, callback) {
const body = Table.formatMetadata_(metadata);
super.setMetadata(body, callback);
}
getIamPolicy(optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
if (typeof options.requestedPolicyVersion === 'number' &&
options.requestedPolicyVersion !== 1) {
throw new Error('Only IAM policy version 1 is supported.');
}
const json = extend(true, {}, { options });
this.request({
method: 'POST',
uri: '/:getIamPolicy',
json,
}, (err, resp) => {
if (err) {
callback(err, null);
return;
}
callback(null, resp);
});
}
setIamPolicy(policy, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
if (policy.version && policy.version !== 1) {
throw new Error('Only IAM policy version 1 is supported.');
}
const json = extend(true, {}, options, { policy });
this.request({
method: 'POST',
uri: '/:setIamPolicy',
json,
}, (err, resp) => {
if (err) {
callback(err, null);
return;
}
callback(null, resp);
});
}
testIamPermissions(permissions, callback) {
permissions = arrify(permissions);
const json = extend(true, {}, { permissions });
this.request({
method: 'POST',
uri: '/:testIamPermissions',
json,
}, (err, resp) => {
if (err) {
callback(err, null);
return;
}
callback(null, resp);
});
}
}
exports.Table = Table;
/*! Developer Documentation
*
* These methods can be auto-paginated.
*/
paginator_1.paginator.extend(Table, ['getRows']);
/*! Developer Documentation
*
* All async methods (except for streams) will return a Promise in the event
* that a callback is omitted.
*/
(0, promisify_1.promisifyAll)(Table);
//# sourceMappingURL=table.js.map