@google-cloud/spanner
Version:
Cloud Spanner Client Library for Node.js
404 lines • 15.2 kB
JavaScript
"use strict";
/*!
* Copyright 2016 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.PartialResultStream = void 0;
exports.partialResultStream = partialResultStream;
const service_1 = require("./common-grpc/service");
const checkpointStream = require("checkpoint-stream");
const eventsIntercept = require("events-intercept");
const is = require("is");
const mergeStream = require("merge-stream");
const stream_1 = require("stream");
const streamEvents = require("stream-events");
const google_gax_1 = require("google-gax");
const transaction_runner_1 = require("./transaction-runner");
const codec_1 = require("./codec");
const protos_1 = require("../protos/protos");
const stream = require("stream");
/**
* The PartialResultStream transforms partial result set objects into Row
* objects.
*
* @class
* @extends {Transform}
*
* @param {RowOptions} [options] The row options.
*/
class PartialResultStream extends stream_1.Transform {
_destroyed;
_fields;
_options;
_pendingValue;
_pendingValueForResume;
_values;
_numPushFailed = 0;
constructor(options = {}) {
super({ objectMode: true });
this._destroyed = false;
this._options = Object.assign({ maxResumeRetries: 20 }, options);
this._values = [];
}
/**
* Destroys the stream.
*
* @param {Error} [err] Optional error to destroy stream with.
*/
destroy(err) {
if (this._destroyed) {
return this;
}
this._destroyed = true;
process.nextTick(() => {
if (err) {
this.emit('error', err);
}
this.emit('close');
});
return this;
}
/**
* Processes each chunk.
*
* @private
*
* @param {object} chunk The partial result set.
* @param {string} encoding Chunk encoding (Not used in object streams).
* @param {function} next Function to be called upon completion.
*/
_transform(chunk, enc, next) {
this.emit('response', chunk);
if (chunk.stats) {
this.emit('stats', chunk.stats);
}
if (!this._fields && chunk.metadata) {
this._fields = chunk.metadata.rowType
.fields;
}
let res = true;
if (!is.empty(chunk.values)) {
res = this._addChunk(chunk);
}
if (res) {
next();
}
else {
// Wait a little before we push any more data into the pipeline as a
// component downstream has indicated that a break is needed. Pause the
// request stream to prevent it from filling up the buffer while we are
// waiting.
// The stream will initially pause for 2ms, and then double the pause time
// for each new pause.
const initialPauseMs = 2;
setTimeout(() => {
this._tryResume(next, 2 * initialPauseMs);
}, initialPauseMs);
}
}
_tryResume(next, timeout) {
// Try to push an empty chunk to check whether more data can be accepted.
if (this.push(undefined)) {
this._numPushFailed = 0;
this.emit('resumed');
next();
}
else {
// Downstream returned false indicating that it is still not ready for
// more data.
this._numPushFailed++;
if (this._numPushFailed === this._options.maxResumeRetries) {
this.destroy(new Error(`Stream is still not ready to receive data after ${this._numPushFailed} attempts to resume.`));
return;
}
setTimeout(() => {
const nextTimeout = Math.min(timeout * 2, 1024);
this._tryResume(next, nextTimeout);
}, timeout);
}
}
_resetPendingValues() {
if (this._pendingValueForResume) {
this._pendingValue = this._pendingValueForResume;
}
else {
delete this._pendingValue;
}
}
/**
* Manages any chunked values.
*
* @private
*
* @param {object} chunk The partial result set.
*/
_addChunk(chunk) {
const values = chunk.values.map(service_1.GrpcService.decodeValue_);
// If we have a chunk to merge, merge the values now.
if (this._pendingValue) {
const currentField = this._values.length % this._fields.length;
const field = this._fields[currentField];
const merged = PartialResultStream.merge(field.type, this._pendingValue, values.shift());
values.unshift(...merged);
delete this._pendingValue;
}
// If the chunk is chunked, store the last value for merging with the next
// chunk to be processed.
if (chunk.chunkedValue) {
this._pendingValue = values.pop();
if (_hasResumeToken(chunk)) {
this._pendingValueForResume = this._pendingValue;
}
}
else if (_hasResumeToken(chunk)) {
delete this._pendingValueForResume;
}
let res = true;
values.forEach(value => {
res = this._addValue(value) && res;
if (!res) {
this.emit('paused');
}
});
return res;
}
/**
* Manages complete values, pushing a completed row into the stream once all
* values have been received.
*
* @private
*
* @param {*} value The complete value.
*/
_addValue(value) {
const values = this._values;
values.push(value);
if (values.length !== this._fields.length) {
return true;
}
this._values = [];
const row = this._createRow(values);
if (this._options.json) {
return this.push(row.toJSON(this._options.jsonOptions));
}
return this.push(row);
}
/**
* Converts an array of values into a row.
*
* @private
*
* @param {Array.<*>} values The row values.
* @returns {Row}
*/
_createRow(values) {
const fields = values.map((value, index) => {
const { name, type } = this._fields[index];
const columnMetadata = this._options.columnsMetadata?.[name];
return {
name,
value: codec_1.codec.decode(value, type, columnMetadata),
};
});
Object.defineProperty(fields, 'toJSON', {
value: (options) => {
return codec_1.codec.convertFieldsToJson(fields, options);
},
});
return fields;
}
/**
* Attempts to merge chunked values together.
*
* @static
* @private
*
* @param {object} type The value type.
* @param {*} head The head of the combined value.
* @param {*} tail The tail of the combined value.
* @returns {Array.<*>}
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
static merge(type, head, tail) {
if (type.code === protos_1.google.spanner.v1.TypeCode.ARRAY ||
type.code === 'ARRAY' ||
type.code === protos_1.google.spanner.v1.TypeCode.STRUCT ||
type.code === 'STRUCT') {
if (head === null || tail === null) {
return [head, tail];
}
return [PartialResultStream.mergeLists(type, head, tail)];
}
if (is.string(head) && is.string(tail)) {
return [head + tail];
}
return [head, tail];
}
/**
* Attempts to merge chunked lists together.
*
* @static
* @private
*
* @param {object} type The list type.
* @param {Array.<*>} head The beginning of the list.
* @param {Array.<*>} tail The end of the list.
* @returns {Array.<*>}
*/
static mergeLists(type, head, tail) {
let listType;
if (type.code === 'ARRAY' ||
type.code === protos_1.google.spanner.v1.TypeCode.ARRAY) {
listType = type.arrayElementType;
}
else {
listType = type.structType.fields[head.length - 1]
.type;
}
const merged = PartialResultStream.merge(listType, head.pop(), tail.shift());
return [...head, ...merged, ...tail];
}
}
exports.PartialResultStream = PartialResultStream;
/**
* Rows returned from queries may be chunked, requiring them to be stitched
* together. This function returns a stream that will properly assemble these
* rows, as well as retry after an error. Rows are only emitted if they hit a
* "checkpoint", which is when a `resumeToken` is returned from the API. Without
* that token, it's unsafe for the query to be retried, as we wouldn't want to
* emit the same data multiple times.
*
* @private
*
* @param {RequestFunction} requestFn The function that makes an API request. It
* will receive one argument, `resumeToken`, which should be used however is
* necessary to send to the API for additional requests.
* @param {RowOptions} [options] Options for formatting rows.
* @returns {PartialResultStream}
*/
function partialResultStream(requestFn, options) {
const retryableCodes = [google_gax_1.grpc.status.UNAVAILABLE];
const maxQueued = 10;
let lastResumeToken;
let lastRequestStream;
const startTime = Date.now();
const timeout = options?.gaxOptions?.timeout ?? Infinity;
// mergeStream allows multiple streams to be connected into one. This is good;
// if we need to retry a request and pipe more data to the user's stream.
// We also add an additional stream that can be used to flush any remaining
// items in the checkpoint stream that have been received, and that did not
// contain a resume token.
const requestsStream = mergeStream();
const flushStream = new stream.PassThrough({ objectMode: true });
requestsStream.add(flushStream);
const partialRSStream = new PartialResultStream(options);
const userStream = streamEvents(partialRSStream);
// We keep track of the number of PartialResultSets that did not include a
// resume token, as that is an indication whether it is safe to retry the
// stream halfway.
let withoutCheckpointCount = 0;
const batchAndSplitOnTokenStream = checkpointStream.obj({
maxQueued,
isCheckpointFn: (chunk) => {
const withCheckpoint = _hasResumeToken(chunk);
if (withCheckpoint) {
withoutCheckpointCount = 0;
}
else {
withoutCheckpointCount++;
}
return withCheckpoint;
},
});
// This listener ensures that the last request that executed successfully
// after one or more retries will end the requestsStream.
const endListener = () => {
setImmediate(() => {
// Push a fake PartialResultSet without any values but with a resume token
// into the stream to ensure that the checkpoint stream is emptied, and
// then push `null` to end the stream.
flushStream.push({ resumeToken: '_' });
flushStream.push(null);
requestsStream.end();
});
};
const makeRequest = () => {
if (is.defined(lastResumeToken) && lastResumeToken.length > 0) {
partialRSStream._resetPendingValues();
}
lastRequestStream = requestFn(lastResumeToken);
lastRequestStream.on('end', endListener);
requestsStream.add(lastRequestStream);
};
const retry = (err) => {
const elapsed = Date.now() - startTime;
if (elapsed >= timeout) {
// The timeout has reached so this will flush any rows the
// checkpoint stream has queued. After that, we will destroy the
// user's stream with the Deadline exceeded error.
setImmediate(() => batchAndSplitOnTokenStream.destroy(new transaction_runner_1.DeadlineError(err)));
return;
}
if (!(err.code &&
(retryableCodes.includes(err.code) || (0, transaction_runner_1.isRetryableInternalError)(err))) ||
// If we have received too many chunks without a resume token, it is not
// safe to retry.
withoutCheckpointCount > maxQueued) {
// This is not a retryable error so this will flush any rows the
// checkpoint stream has queued. After that, we will destroy the
// user's stream with the same error.
setImmediate(() => batchAndSplitOnTokenStream.destroy(err));
return;
}
if (lastRequestStream) {
lastRequestStream.removeListener('end', endListener);
lastRequestStream.destroy();
}
// Delay the retry until all the values that are already in the stream
// pipeline have been handled. This ensures that the checkpoint stream is
// reset to the correct point. Calling .reset() directly here could cause
// any values that are currently in the pipeline and that have not been
// handled yet, to be pushed twice into the entire stream.
setImmediate(() => {
// Empty queued rows on the checkpoint stream (will not emit them to user).
batchAndSplitOnTokenStream.reset();
makeRequest();
});
};
userStream.once('reading', makeRequest);
eventsIntercept.patch(requestsStream);
// need types for events-intercept
// eslint-disable-next-line @typescript-eslint/no-explicit-any
requestsStream.intercept('error', err =>
// Retry __after__ all pending data has been processed to ensure that the
// checkpoint stream is reset at the correct position.
setImmediate(() => retry(err)));
return (requestsStream
.pipe(batchAndSplitOnTokenStream)
// If we get this error, the checkpoint stream has flushed any rows
// it had queued. We can now destroy the user's stream, as our retry
// attempts are over.
.on('error', (err) => userStream.destroy(err))
.on('checkpoint', (row) => {
lastResumeToken = row.resumeToken;
})
.pipe(userStream)
.on('paused', () => requestsStream.pause())
.on('resumed', () => requestsStream.resume()));
}
function _hasResumeToken(chunk) {
return is.defined(chunk.resumeToken) && chunk.resumeToken.length > 0;
}
//# sourceMappingURL=partial-result-stream.js.map