@google-cloud/datastore
Version:
Cloud Datastore Client Library for Node.js
807 lines • 30.3 kB
JavaScript
;
/*!
* Copyright 2014 Google LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.DatastoreRequest = exports.TransactionState = exports.transactionExpiredError = void 0;
exports.getTransactionRequest = getTransactionRequest;
const promisify_1 = require("@google-cloud/promisify");
const arrify = require("arrify");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const concat = require('concat-stream');
const extend = require("extend");
const split_array_stream_1 = require("split-array-stream");
const stream_1 = require("stream");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const streamEvents = require('stream-events');
exports.transactionExpiredError = 'This transaction has already expired.';
// Import the clients for each version supported by this package.
const gapic = Object.freeze({
v1: require('./v1'),
});
const entity_1 = require("./entity");
const query_1 = require("./query");
const _1 = require(".");
const google_gax_1 = require("google-gax");
const gax = require("google-gax");
const root = gax.protobuf.loadSync('google/protobuf/struct.proto');
const Struct = root.lookupType('Struct');
// This function decodes Struct proto values
function decodeStruct(structValue) {
return google_gax_1.serializer.toProto3JSON(Struct.fromObject(structValue));
}
// This function gets a RunQueryInfo object that contains explain metrics that
// were returned from the server.
function getInfoFromStats(resp) {
// Decode struct values stored in planSummary and executionStats
const explainMetrics = {};
if (resp &&
resp.explainMetrics &&
resp.explainMetrics.planSummary &&
resp.explainMetrics.planSummary.indexesUsed) {
Object.assign(explainMetrics, {
planSummary: {
indexesUsed: resp.explainMetrics.planSummary.indexesUsed.map((index) => decodeStruct(index)),
},
});
}
if (resp && resp.explainMetrics && resp.explainMetrics.executionStats) {
const executionStats = {};
{
const resultsReturned = resp.explainMetrics.executionStats.resultsReturned;
if (resultsReturned) {
Object.assign(executionStats, {
resultsReturned: typeof resultsReturned === 'string'
? parseInt(resultsReturned)
: resultsReturned,
});
}
}
{
const executionDuration = resp.explainMetrics.executionStats.executionDuration;
if (executionDuration) {
Object.assign(executionStats, {
executionDuration: typeof executionDuration === 'string'
? parseInt(executionDuration)
: executionDuration,
});
}
}
{
const readOperations = resp.explainMetrics.executionStats.readOperations;
if (readOperations) {
Object.assign(executionStats, {
readOperations: typeof readOperations === 'string'
? parseInt(readOperations)
: readOperations,
});
}
}
{
const debugStats = resp.explainMetrics.executionStats.debugStats;
if (debugStats) {
Object.assign(executionStats, { debugStats: decodeStruct(debugStats) });
}
}
Object.assign(explainMetrics, { executionStats });
}
if (explainMetrics.planSummary || explainMetrics.executionStats) {
return { explainMetrics };
}
return {};
}
const readTimeAndConsistencyError = 'Read time and read consistency cannot both be specified.';
// Write function to check for readTime and readConsistency.
function throwOnReadTimeAndConsistency(options) {
if (options.readTime && options.consistency) {
throw new Error(readTimeAndConsistencyError);
}
}
/**
* A map of read consistency values to proto codes.
*
* @type {object}
* @private
*/
const CONSISTENCY_PROTO_CODE = {
eventual: 2,
strong: 1,
};
/**
* By default a DatastoreRequest is in the NOT_TRANSACTION state. If the
* DatastoreRequest is a Transaction object, then initially it will be in
* the NOT_STARTED state, but then the state will become IN_PROGRESS after the
* transaction has started.
*/
var TransactionState;
(function (TransactionState) {
TransactionState[TransactionState["NOT_TRANSACTION"] = 0] = "NOT_TRANSACTION";
TransactionState[TransactionState["NOT_STARTED"] = 1] = "NOT_STARTED";
TransactionState[TransactionState["IN_PROGRESS"] = 2] = "IN_PROGRESS";
TransactionState[TransactionState["EXPIRED"] = 3] = "EXPIRED";
})(TransactionState || (exports.TransactionState = TransactionState = {}));
/**
* Handles request logic for Datastore API operations.
*
* Creates requests to the Datastore endpoint. Designed to be inherited by
* the {@link Datastore} and {@link Transaction} classes.
*
* @class
*/
class DatastoreRequest {
id;
requests_;
requestCallbacks_;
datastore;
state = TransactionState.NOT_TRANSACTION;
/**
* Format a user's input to mutation methods. This will create a deep clone of
* the input, as well as allow users to pass an object in the format of an
* entity.
*
* Both of the following formats can be supplied supported:
*
* datastore.save({
* key: datastore.key('Kind'),
* data: { foo: 'bar' }
* }, (err) => {})
*
* const entity = { foo: 'bar' }
* entity[datastore.KEY] = datastore.key('Kind')
* datastore.save(entity, (err) => {})
*
* @internal
*
* @see {@link https://github.com/GoogleCloudPlatform/google-cloud-node/issues/1803}
*
* @param {object} obj The user's input object.
*/
static prepareEntityObject_(obj) {
const entityObject = extend(true, {}, obj);
// Entity objects are also supported.
if (obj[entity_1.entity.KEY_SYMBOL]) {
return {
key: obj[entity_1.entity.KEY_SYMBOL],
data: entityObject,
};
}
return entityObject;
}
allocateIds(key, options, callback) {
if (entity_1.entity.isKeyComplete(key)) {
throw new Error('An incomplete key should be provided.');
}
options = typeof options === 'number' ? { allocations: options } : options;
this.request_({
client: 'DatastoreClient',
method: 'allocateIds',
reqOpts: {
keys: new Array(options.allocations).fill(entity_1.entity.keyToKeyProto(key)),
},
gaxOpts: options.gaxOptions,
}, (err, resp) => {
if (err) {
callback(err, null, resp);
return;
}
const keys = arrify(resp.keys).map(entity_1.entity.keyFromKeyProto);
callback(null, keys, resp);
});
}
/* This throws an error if the transaction has already expired.
*
*/
checkExpired() {
if (this.state === TransactionState.EXPIRED) {
throw Error(exports.transactionExpiredError);
}
}
/**
* Retrieve the entities as a readable object stream.
*
* @throws {Error} If at least one Key object is not provided.
* @throws {Error} If read time and read consistency cannot both be specified.
*
* @param {Key|Key[]} keys Datastore key object(s).
* @param {object} [options] Optional configuration. See {@link Datastore#get}
* for a complete list of options.
*
* @example
* ```
* const keys = [
* datastore.key(['Company', 123]),
* datastore.key(['Product', 'Computer'])
* ];
*
* datastore.createReadStream(keys)
* .on('error', (err) => {})
* .on('data', (entity) => {
* // entity is an entity object.
* })
* .on('end', () => {
* // All entities retrieved.
* });
* ```
*/
createReadStream(keys, options = {}) {
keys = arrify(keys).map(entity_1.entity.keyToKeyProto);
if (keys.length === 0) {
throw new Error('At least one Key object is required.');
}
this.checkExpired();
throwOnReadTimeAndConsistency(options);
const reqOpts = this.getRequestOptions(options);
throwOnTransactionErrors(this, reqOpts);
const makeRequest = (keys) => {
Object.assign(reqOpts, { keys });
this.request_({
client: 'DatastoreClient',
method: 'lookup',
reqOpts,
gaxOpts: options.gaxOptions,
}, (err, resp) => {
this.parseTransactionResponse(resp);
if (err) {
stream.destroy(err);
return;
}
let entities = [];
try {
entities = entity_1.entity.formatArray(resp.found, options.wrapNumbers);
}
catch (err) {
stream.destroy(err);
return;
}
const nextKeys = (resp.deferred || [])
.map(entity_1.entity.keyFromKeyProto)
.map(entity_1.entity.keyToKeyProto);
(0, split_array_stream_1.split)(entities, stream)
.then(streamEnded => {
if (streamEnded) {
return;
}
if (nextKeys.length > 0) {
makeRequest(nextKeys);
return;
}
stream.push(null);
})
.catch(err => {
throw err;
});
});
};
const stream = streamEvents(new stream_1.Transform({ objectMode: true }));
stream.once('reading', () => {
makeRequest(keys);
});
return stream;
}
delete(keys, gaxOptionsOrCallback, cb) {
const gaxOptions = typeof gaxOptionsOrCallback === 'object' ? gaxOptionsOrCallback : {};
const callback = typeof gaxOptionsOrCallback === 'function' ? gaxOptionsOrCallback : cb;
const reqOpts = {
mutations: arrify(keys).map(key => {
return {
delete: entity_1.entity.keyToKeyProto(key),
};
}),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
};
if (this.id) {
this.requests_.push(reqOpts);
return;
}
this.request_({
client: 'DatastoreClient',
method: 'commit',
reqOpts,
gaxOpts: gaxOptions,
}, callback);
}
get(keys, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' && optionsOrCallback
? optionsOrCallback
: {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
try {
this.createReadStream(keys, options)
.on('error', callback)
.pipe(concat((results) => {
const isSingleLookup = !Array.isArray(keys);
callback(null, isSingleLookup ? results[0] : results);
}));
}
catch (err) {
callback(err);
}
}
/**
* This function saves results from a successful beginTransaction call.
*
* @param {object} [response] The response from a call to
* begin a transaction that completed successfully.
*
**/
parseTransactionResponse(resp) {
if (resp && resp.transaction && Buffer.byteLength(resp.transaction) > 0) {
this.id = resp.transaction;
this.state = TransactionState.IN_PROGRESS;
}
}
runAggregationQuery(query, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
if (this.state === TransactionState.EXPIRED) {
callback(new Error(exports.transactionExpiredError));
return;
}
if (options.readTime && options.consistency) {
callback(new Error(readTimeAndConsistencyError));
return;
}
query.query = extend(true, new query_1.Query(), query.query);
let queryProto;
try {
queryProto = entity_1.entity.queryToQueryProto(query.query);
}
catch (e) {
// using setImmediate here to make sure this doesn't throw a
// synchronous error
setImmediate(callback, e);
return;
}
let sharedQueryOpts;
try {
sharedQueryOpts = this.getQueryOptions(query.query, options);
throwOnTransactionErrors(this, sharedQueryOpts);
}
catch (error) {
callback(error);
return;
}
const aggregationQueryOptions = {
nestedQuery: queryProto,
aggregations: query.toProto(),
};
const reqOpts = Object.assign(sharedQueryOpts, {
aggregationQuery: aggregationQueryOptions,
});
this.request_({
client: 'DatastoreClient',
method: 'runAggregationQuery',
reqOpts,
gaxOpts: options.gaxOptions,
}, (err, res) => {
const info = getInfoFromStats(res);
this.parseTransactionResponse(res);
if (res && res.batch) {
const results = res.batch.aggregationResults;
const finalResults = results
.map((aggregationResult) => aggregationResult.aggregateProperties)
.map((aggregateProperties) => Object.fromEntries(new Map(Object.keys(aggregateProperties).map(key => [
key,
entity_1.entity.decodeValueProto(aggregateProperties[key]),
]))));
callback(err, finalResults, info);
}
else {
callback(err, [], info);
}
});
}
runQuery(query, optionsOrCallback, cb) {
const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
let info;
try {
this.runQueryStream(query, options)
.on('error', callback)
.on('info', info_ => {
info = info_;
})
.pipe(concat((results) => {
callback(null, results, info);
}));
}
catch (err) {
callback(err);
}
}
/**
* Get a list of entities as a readable object stream.
*
* See {@link Datastore#runQuery} for a list of all available options.
*
* @param {Query} query A Query object
* @param {object} [options] Optional configuration.
* @param {object} [options.gaxOptions] Request configuration options, outlined
* here: https://googleapis.github.io/gax-nodejs/global.html#CallOptions.
*
* @throws {Error} If read time and read consistency cannot both be specified.
*
* @example
* ```
* datastore.runQueryStream(query)
* .on('error', console.error)
* .on('data', (entity) => {
* // Access the Key object for this entity.
* const key = entity[datastore.KEY];
* })
* .on('info', (info) => {})
* .on('end', () => {
* // All entities retrieved.
* });
*
* //-
* // If you anticipate many results, you can end a stream early to prevent
* // unnecessary processing and API requests.
* //-
* datastore.runQueryStream(query)
* .on('data', (entity) => {
* this.end();
* });
* ```
*/
runQueryStream(query, options = {}) {
this.checkExpired();
throwOnReadTimeAndConsistency(options);
query = extend(true, new query_1.Query(), query);
const sharedQueryOpts = this.getQueryOptions(query, options);
throwOnTransactionErrors(this, sharedQueryOpts);
const makeRequest = (query) => {
let queryProto;
try {
queryProto = entity_1.entity.queryToQueryProto(query);
}
catch (e) {
// using setImmediate here to make sure this doesn't throw a
// synchronous error
setImmediate(onResultSet, e);
return;
}
const reqOpts = sharedQueryOpts;
reqOpts.query = queryProto;
this.request_({
client: 'DatastoreClient',
method: 'runQuery',
reqOpts,
gaxOpts: options.gaxOptions,
}, onResultSet);
};
const onResultSet = (err, resp) => {
this.parseTransactionResponse(resp);
if (err) {
stream.destroy(err);
return;
}
if (!resp.batch) {
// If there are no results then send any stats back and end the stream.
stream.emit('info', getInfoFromStats(resp));
stream.push(null);
return;
}
const info = Object.assign(getInfoFromStats(resp), {
moreResults: resp.batch.moreResults,
});
if (resp.batch.endCursor) {
info.endCursor = resp.batch.endCursor.toString('base64');
}
let entities = [];
if (resp.batch.entityResults) {
try {
entities = entity_1.entity.formatArray(resp.batch.entityResults, options.wrapNumbers);
}
catch (err) {
stream.destroy(err);
return;
}
}
// Emit each result right away, then get the rest if necessary.
(0, split_array_stream_1.split)(entities, stream)
.then(streamEnded => {
if (streamEnded) {
return;
}
if (resp.batch.moreResults !== 'NOT_FINISHED') {
stream.emit('info', info);
stream.push(null);
return;
}
// The query is "NOT_FINISHED". Get the rest of the results.
const offset = query.offsetVal === -1 ? 0 : query.offsetVal;
query
.start(info.endCursor)
.offset(offset - resp.batch.skippedResults);
const limit = query.limitVal;
if (limit && limit > -1) {
query.limit(limit - resp.batch.entityResults.length);
}
makeRequest(query);
})
.catch(err => {
throw err;
});
};
const stream = streamEvents(new stream_1.Transform({ objectMode: true }));
stream.once('reading', () => {
makeRequest(query);
});
return stream;
}
/**
* Gets request options from a RunQueryStream options configuration
*
* @param {RunQueryStreamOptions} [options] The RunQueryStream options configuration
*/
getRequestOptions(options) {
const sharedQueryOpts = {};
if (isTransaction(this)) {
if (this.state === TransactionState.NOT_STARTED) {
if (sharedQueryOpts.readOptions === undefined) {
sharedQueryOpts.readOptions = {};
}
sharedQueryOpts.readOptions.newTransaction = getTransactionRequest(this, {});
sharedQueryOpts.readOptions.consistencyType = 'newTransaction';
}
}
if (options.consistency) {
const code = CONSISTENCY_PROTO_CODE[options.consistency.toLowerCase()];
if (sharedQueryOpts.readOptions === undefined) {
sharedQueryOpts.readOptions = {};
}
sharedQueryOpts.readOptions.readConsistency = code;
}
if (options.readTime) {
if (sharedQueryOpts.readOptions === undefined) {
sharedQueryOpts.readOptions = {};
}
const readTime = options.readTime;
const seconds = readTime / 1000;
sharedQueryOpts.readOptions.readTime = {
seconds: Math.floor(seconds),
};
}
return sharedQueryOpts;
}
/**
* Gets request options from a RunQueryStream options configuration
*
* @param {Query} [query] A Query object
* @param {RunQueryStreamOptions} [options] The RunQueryStream options configuration
*/
getQueryOptions(query, options = {}) {
const sharedQueryOpts = this.getRequestOptions(options);
if (options.explainOptions) {
sharedQueryOpts.explainOptions = options.explainOptions;
}
if (query.namespace) {
sharedQueryOpts.partitionId = {
namespaceId: query.namespace,
};
}
return sharedQueryOpts;
}
merge(entities, callback) {
const transaction = this.datastore.transaction();
transaction.run(async (err) => {
if (err) {
try {
await transaction.rollback();
}
catch (error) {
// Provide the error & API response from the failed run to the user.
// Even a failed rollback should be transparent.
// RE: https://github.com/GoogleCloudPlatform/gcloud-node/pull/1369#discussion_r66833976
}
callback(err);
return;
}
try {
await Promise.all(arrify(entities).map(async (objEntity) => {
const obj = DatastoreRequest.prepareEntityObject_(objEntity);
const [data] = await transaction.get(obj.key);
obj.method = 'upsert';
obj.data = Object.assign({}, data, obj.data);
transaction.save(obj);
}));
const [response] = await transaction.commit();
callback(null, response);
}
catch (err) {
try {
await transaction.rollback();
}
catch (error) {
// Provide the error & API response from the failed commit to the user.
// Even a failed rollback should be transparent.
// RE: https://github.com/GoogleCloudPlatform/gcloud-node/pull/1369#discussion_r66833976
}
callback(err);
}
});
}
/**
* Builds a request and sends it to the Gapic Layer.
*
* @param {object} config Configuration object.
* @param {function} callback The callback function.
*
* @private
*/
prepareGaxRequest_(config, callback) {
const datastore = this.datastore;
const isTransaction = this.id ? true : false;
const method = config.method;
const reqOpts = extend(true, {}, config.reqOpts);
// Set properties to indicate if we're in a transaction or not.
if (method === 'commit') {
if (isTransaction) {
reqOpts.mode = 'TRANSACTIONAL';
reqOpts.transaction = this.id;
}
else {
reqOpts.mode = 'NON_TRANSACTIONAL';
}
}
if (datastore.options && datastore.options.databaseId) {
reqOpts.databaseId = datastore.options.databaseId;
}
if (method === 'rollback') {
reqOpts.transaction = this.id;
}
throwOnTransactionErrors(this, reqOpts);
if (isTransaction &&
(method === 'lookup' ||
method === 'runQuery' ||
method === 'runAggregationQuery')) {
if (reqOpts.readOptions) {
Object.assign(reqOpts.readOptions, { transaction: this.id });
}
else {
reqOpts.readOptions = {
transaction: this.id,
};
}
}
datastore.auth.getProjectId((err, projectId) => {
if (err) {
callback(err);
return;
}
const clientName = config.client;
if (!datastore.clients_.has(clientName)) {
datastore.clients_.set(clientName, new gapic.v1[clientName](datastore.options));
}
const gaxClient = datastore.clients_.get(clientName);
reqOpts.projectId = projectId;
const gaxOpts = extend(true, {}, config.gaxOpts, {
headers: {
'google-cloud-resource-prefix': `projects/${projectId}`,
},
});
const requestFn = gaxClient[method].bind(gaxClient, reqOpts, gaxOpts);
callback(null, requestFn);
});
}
request_(config, callback) {
this.prepareGaxRequest_(config, (err, requestFn) => {
if (err) {
callback(err);
return;
}
requestFn(callback);
});
}
/**
* Make a request as a stream.
*
* @param {object} config Configuration object.
* @param {object} config.gaxOpts GAX options.
* @param {string} config.client The name of the gax client.
* @param {string} config.method The gax method to call.
* @param {object} config.reqOpts Request options.
*/
requestStream_(config) {
let gaxStream;
const stream = streamEvents(new stream_1.PassThrough({ objectMode: true }));
stream.abort = () => {
if (gaxStream && gaxStream.cancel) {
gaxStream.cancel();
}
};
stream.once('reading', () => {
this.prepareGaxRequest_(config, (err, requestFn) => {
if (err) {
stream.destroy(err);
return;
}
gaxStream = requestFn();
gaxStream
.on('error', stream.destroy.bind(stream))
.on('response', stream.emit.bind(stream, 'response'))
.pipe(stream);
});
});
return stream;
}
}
exports.DatastoreRequest = DatastoreRequest;
/**
* Check to see if a request is a Transaction
*
* @param {DatastoreRequest} request The Datastore request object
*
*/
function isTransaction(request) {
return request instanceof _1.Transaction;
}
/**
* Throw an error if read options are not properly specified.
*
* @param {DatastoreRequest} request The Datastore request object
* @param {SharedQueryOptions} options The Query options
*
*/
function throwOnTransactionErrors(request, options) {
const isTransaction = request.id ? true : false;
if (isTransaction ||
(options.readOptions && options.readOptions.newTransaction)) {
if (options.readOptions && options.readOptions.readConsistency) {
throw new Error('Read consistency cannot be specified in a transaction.');
}
if (options.readOptions && options.readOptions.readTime) {
throw new Error('Read time cannot be specified in a transaction.');
}
}
}
/**
* This function gets transaction request options used for defining a
* request to create a new transaction on the server.
*
* @param {Transaction} transaction The transaction for which the request will be made.
* @param {RunOptions} options Custom options that will be used to create the request.
*/
function getTransactionRequest(transaction, options) {
// If transactionOptions are provide then they will be used.
// Otherwise, options passed into this function are used and when absent
// options that exist on Transaction are used.
return options.transactionOptions // If transactionOptions is specified:
? options.transactionOptions.readOnly // Use readOnly on transactionOptions
? { readOnly: {} }
: options.transactionOptions.id // Use retry transaction if specified:
? { readWrite: { previousTransaction: options.transactionOptions.id } }
: {}
: options.readOnly || transaction.readOnly // If transactionOptions not set:
? { readOnly: {} } // Create a readOnly transaction if readOnly option set
: options.transactionId || transaction.id
? {
// Create readWrite transaction with a retry transaction set
readWrite: {
previousTransaction: options.transactionId || transaction.id,
},
}
: {}; // Request will be readWrite with no retry transaction set;
}
/*! Developer Documentation
*
* All async methods (except for streams) will return a Promise in the event
* that a callback is omitted.
*/
(0, promisify_1.promisifyAll)(DatastoreRequest, {
exclude: ['checkExpired', 'getQueryOptions', 'getRequestOptions'],
});
//# sourceMappingURL=request.js.map