@google-cloud/spanner
Version:
Cloud Spanner Client Library for Node.js
476 lines • 18 kB
JavaScript
"use strict";
/*!
* Copyright 2016 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.PartialResultStream = void 0;
exports.partialResultStream = partialResultStream;
const service_1 = require("./common-grpc/service");
const checkpointStream = require("checkpoint-stream");
const eventsIntercept = require("events-intercept");
const mergeStream = require("merge-stream");
const stream_1 = require("stream");
const streamEvents = require("stream-events");
const google_gax_1 = require("google-gax");
const transaction_runner_1 = require("./transaction-runner");
const codec_1 = require("./codec");
const spanner_api_1 = require("@google-cloud/spanner-api");
var google = spanner_api_1.protos.google;
const stream = require("stream");
const helper_1 = require("./helper");
const originalDecode = codec_1.codec.decode;
const originalConvertFieldsToJson = codec_1.codec.convertFieldsToJson;
/**
* The PartialResultStream transforms partial result set objects into Row
* objects.
*
* @class
* @extends {Transform}
*
* @param {RowOptions} [options] The row options.
*/
class PartialResultStream extends stream_1.Transform {
_destroyed;
_fields;
_decoders;
_options;
_pendingValue;
_pendingValueForResume;
_values;
_numPushFailed = 0;
constructor(options = {}) {
super({ objectMode: true });
this._destroyed = false;
this._options = Object.assign({ maxResumeRetries: 20 }, options);
this._values = [];
}
/**
* Destroys the stream.
*
* @param {Error} [err] Optional error to destroy stream with.
*/
destroy(err) {
if (this._destroyed) {
return this;
}
this._destroyed = true;
process.nextTick(() => {
if (err) {
this.emit('error', err);
}
this.emit('close');
});
return this;
}
/**
* Processes each chunk.
*
* @private
*
* @param {object} chunk The partial result set.
* @param {string} encoding Chunk encoding (Not used in object streams).
* @param {function} next Function to be called upon completion.
*/
_transform(chunk, enc, next) {
this.emit('response', chunk);
if (chunk.stats) {
this.emit('stats', chunk.stats);
}
if (!this._fields && chunk.metadata) {
this._fields = chunk.metadata.rowType
.fields;
this._decoders = this._fields.map(({ name, type }) => {
const columnMetadata = this._options.columnsMetadata &&
name !== null &&
name !== undefined &&
Object.prototype.hasOwnProperty.call(this._options.columnsMetadata, name)
? this._options.columnsMetadata[name]
: undefined;
if (codec_1.codec.decode !== originalDecode) {
return val => codec_1.codec.decode(val, type, columnMetadata);
}
return codec_1.codec.getDecoder(type, columnMetadata, this._options.json ? this._options.jsonOptions || {} : undefined);
});
}
let res = true;
if (!(0, helper_1.isEmpty)(chunk.values)) {
try {
res = this._addChunk(chunk);
}
catch (err) {
next(err);
return;
}
}
if (chunk.last) {
this.push(null);
return;
}
if (res) {
next();
}
else {
// Wait a little before we push any more data into the pipeline as a
// component downstream has indicated that a break is needed. Pause the
// request stream to prevent it from filling up the buffer while we are
// waiting.
// The stream will initially pause for 2ms, and then double the pause time
// for each new pause.
const initialPauseMs = 2;
setTimeout(() => {
this._tryResume(next, 2 * initialPauseMs);
}, initialPauseMs);
}
}
_tryResume(next, timeout) {
// Try to push an empty chunk to check whether more data can be accepted.
if (this.push(undefined)) {
this._numPushFailed = 0;
this.emit('resumed');
next();
}
else {
// Downstream returned false indicating that it is still not ready for
// more data.
this._numPushFailed++;
if (this._numPushFailed === this._options.maxResumeRetries) {
this.destroy(new Error(`Stream is still not ready to receive data after ${this._numPushFailed} attempts to resume.`));
return;
}
setTimeout(() => {
const nextTimeout = Math.min(timeout * 2, 1024);
this._tryResume(next, nextTimeout);
}, timeout);
}
}
_resetPendingValues() {
if (this._pendingValueForResume) {
this._pendingValue = this._pendingValueForResume;
}
else {
delete this._pendingValue;
}
}
/**
* Manages any chunked values.
*
* @private
*
* @param {object} chunk The partial result set.
*/
_addChunk(chunk) {
const chunkValues = chunk.values;
const numValues = chunkValues.length;
const values = new Array(numValues);
for (let i = 0; i < numValues; i++) {
values[i] = service_1.GrpcService.decodeValue_(chunkValues[i]);
}
// If we have a chunk to merge, merge the values now.
if (this._pendingValue) {
const currentField = this._values.length % this._fields.length;
const field = this._fields[currentField];
const merged = PartialResultStream.merge(field.type, this._pendingValue, values.shift());
values.unshift(...merged);
delete this._pendingValue;
}
// If the chunk is chunked, store the last value for merging with the next
// chunk to be processed.
if (chunk.chunkedValue) {
this._pendingValue = values.pop();
if (_hasResumeToken(chunk)) {
this._pendingValueForResume = this._pendingValue;
}
}
else if (_hasResumeToken(chunk)) {
delete this._pendingValueForResume;
}
let res = true;
const len = values.length;
for (let i = 0; i < len; i++) {
res = this._addValue(values[i]) && res;
if (!res) {
this.emit('paused');
}
}
return res;
}
/**
* Manages complete values, pushing a completed row into the stream once all
* values have been received.
*
* @private
*
* @param {*} value The complete value.
*/
_addValue(value) {
const values = this._values;
values.push(value);
if (values.length !== this._fields.length) {
return true;
}
this._values = [];
const isJsonStubbed = codec_1.codec.convertFieldsToJson !== originalConvertFieldsToJson;
if (this._options.json && !isJsonStubbed) {
return this.push(this._createJsonRow(values));
}
const row = this._createRow(values);
if (this._options.json) {
return this.push(row.toJSON(this._options.jsonOptions));
}
return this.push(row);
}
/**
* Directly creates a plain JSON object from row cell values, bypassing
* Struct, Row, and WrappedNumber class wrappers when possible.
*
* @private
*
* @param {Value[]} values The raw cell values for the current row.
* @returns {Json} The plain JavaScript object representing the row.
*/
_createJsonRow(values) {
const json = {};
const fields = this._fields;
const decoders = this._decoders;
const len = fields.length;
const includeNameless = !!this._options.jsonOptions?.includeNameless;
for (let i = 0; i < len; i++) {
const { name } = fields[i];
if (!name && !includeNameless) {
continue;
}
const fieldName = name ? name : `_${i}`;
try {
json[fieldName] = decoders[i](values[i]);
}
catch (e) {
e.message = [
`Serializing column "${fieldName}" encountered an error: ${e.message}`,
'Call row.toJSON({ wrapNumbers: true }) to receive a custom type.',
].join(' ');
throw e;
}
}
return json;
}
/**
* Converts an array of values into a row.
*
* @private
*
* @param {Array.<*>} values The row values.
* @returns {Row}
*/
_createRow(values) {
const len = values.length;
const fields = new Array(len);
const decoders = this._decoders;
const classFields = this._fields;
for (let i = 0; i < len; i++) {
fields[i] = {
name: classFields[i].name,
value: decoders[i](values[i]),
};
}
Object.defineProperty(fields, 'toJSON', {
value: (options) => {
return codec_1.codec.convertFieldsToJson(fields, options);
},
});
return fields;
}
/**
* Attempts to merge chunked values together.
*
* @static
* @private
*
* @param {object} type The value type.
* @param {*} head The head of the combined value.
* @param {*} tail The tail of the combined value.
* @returns {Array.<*>}
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
static merge(type, head, tail) {
if (type.code === google.spanner.v1.TypeCode.ARRAY ||
type.code === 'ARRAY' ||
type.code === google.spanner.v1.TypeCode.STRUCT ||
type.code === 'STRUCT') {
if (head === null || tail === null) {
return [head, tail];
}
return [PartialResultStream.mergeLists(type, head, tail)];
}
if ((0, helper_1.isString)(head) && (0, helper_1.isString)(tail)) {
return [head + tail];
}
return [head, tail];
}
/**
* Attempts to merge chunked lists together.
*
* @static
* @private
*
* @param {object} type The list type.
* @param {Array.<*>} head The beginning of the list.
* @param {Array.<*>} tail The end of the list.
* @returns {Array.<*>}
*/
static mergeLists(type, head, tail) {
let listType;
if (type.code === 'ARRAY' ||
type.code === google.spanner.v1.TypeCode.ARRAY) {
listType = type.arrayElementType;
}
else {
listType = type.structType.fields[head.length - 1]
.type;
}
const merged = PartialResultStream.merge(listType, head.pop(), tail.shift());
return [...head, ...merged, ...tail];
}
}
exports.PartialResultStream = PartialResultStream;
/**
* Rows returned from queries may be chunked, requiring them to be stitched
* together. This function returns a stream that will properly assemble these
* rows, as well as retry after an error. Rows are only emitted if they hit a
* "checkpoint", which is when a `resumeToken` is returned from the API. Without
* that token, it's unsafe for the query to be retried, as we wouldn't want to
* emit the same data multiple times.
*
* @private
*
* @param {RequestFunction} requestFn The function that makes an API request. It
* will receive one argument, `resumeToken`, which should be used however is
* necessary to send to the API for additional requests.
* @param {RowOptions} [options] Options for formatting rows.
* @returns {PartialResultStream}
*/
function partialResultStream(requestFn, options) {
const retryableCodes = [google_gax_1.grpc.status.UNAVAILABLE];
const maxQueued = 10;
let lastResumeToken;
let lastRequestStream;
const startTime = Date.now();
const timeout = options?.gaxOptions?.timeout ?? Infinity;
// mergeStream allows multiple streams to be connected into one. This is good;
// if we need to retry a request and pipe more data to the user's stream.
// We also add an additional stream that can be used to flush any remaining
// items in the checkpoint stream that have been received, and that did not
// contain a resume token.
const requestsStream = mergeStream();
const flushStream = new stream.PassThrough({ objectMode: true });
requestsStream.add(flushStream);
const partialRSStream = new PartialResultStream(options);
const userStream = streamEvents(partialRSStream);
// We keep track of the number of PartialResultSets that did not include a
// resume token, as that is an indication whether it is safe to retry the
// stream halfway.
let withoutCheckpointCount = 0;
const batchAndSplitOnTokenStream = checkpointStream.obj({
maxQueued,
isCheckpointFn: (chunk) => {
const withCheckpoint = _hasResumeToken(chunk);
if (withCheckpoint) {
withoutCheckpointCount = 0;
}
else {
withoutCheckpointCount++;
}
return withCheckpoint;
},
});
// This listener ensures that the last request that executed successfully
// after one or more retries will end the requestsStream.
const endListener = () => {
setImmediate(() => {
// Push a fake PartialResultSet without any values but with a resume token
// into the stream to ensure that the checkpoint stream is emptied, and
// then push `null` to end the stream.
flushStream.push({ resumeToken: '_' });
flushStream.push(null);
requestsStream.end();
});
};
const makeRequest = () => {
if ((0, helper_1.isDefined)(lastResumeToken) && lastResumeToken.length > 0) {
partialRSStream._resetPendingValues();
}
lastRequestStream = requestFn(lastResumeToken);
lastRequestStream.on('end', endListener);
requestsStream.add(lastRequestStream);
};
const retry = (err) => {
const elapsed = Date.now() - startTime;
if (elapsed >= timeout) {
// The timeout has reached so this will flush any rows the
// checkpoint stream has queued. After that, we will destroy the
// user's stream with the Deadline exceeded error.
setImmediate(() => batchAndSplitOnTokenStream.destroy(new transaction_runner_1.DeadlineError(err)));
return;
}
if (!(err.code &&
(retryableCodes.includes(err.code) || (0, transaction_runner_1.isRetryableInternalError)(err))) ||
// If we have received too many chunks without a resume token, it is not
// safe to retry.
withoutCheckpointCount > maxQueued) {
// This is not a retryable error so this will flush any rows the
// checkpoint stream has queued. After that, we will destroy the
// user's stream with the same error.
setImmediate(() => batchAndSplitOnTokenStream.destroy(err));
return;
}
if (lastRequestStream) {
lastRequestStream.removeListener('end', endListener);
lastRequestStream.destroy();
}
// Delay the retry until all the values that are already in the stream
// pipeline have been handled. This ensures that the checkpoint stream is
// reset to the correct point. Calling .reset() directly here could cause
// any values that are currently in the pipeline and that have not been
// handled yet, to be pushed twice into the entire stream.
setImmediate(() => {
// Empty queued rows on the checkpoint stream (will not emit them to user).
batchAndSplitOnTokenStream.reset();
makeRequest();
});
};
userStream.once('reading', makeRequest);
eventsIntercept.patch(requestsStream);
// need types for events-intercept
// eslint-disable-next-line @typescript-eslint/no-explicit-any
requestsStream.intercept('error', err =>
// Retry __after__ all pending data has been processed to ensure that the
// checkpoint stream is reset at the correct position.
setImmediate(() => retry(err)));
return (requestsStream
.pipe(batchAndSplitOnTokenStream)
// If we get this error, the checkpoint stream has flushed any rows
// it had queued. We can now destroy the user's stream, as our retry
// attempts are over.
.on('error', (err) => userStream.destroy(err))
.on('checkpoint', (row) => {
lastResumeToken = row.resumeToken;
})
.pipe(userStream)
.on('paused', () => requestsStream.pause())
.on('resumed', () => requestsStream.resume()));
}
function _hasResumeToken(chunk) {
return (0, helper_1.isDefined)(chunk.resumeToken) && chunk.resumeToken.length > 0;
}
//# sourceMappingURL=partial-result-stream.js.map