couchbase-index-manager
Version:
Manage Couchbase indexes during the CI/CD process
326 lines • 12.6 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.IndexManager = exports.getKeyspace = exports.DEFAULT_COLLECTION = exports.DEFAULT_SCOPE = void 0;
const httpexecutor_1 = require("couchbase/dist/httpexecutor");
const lodash_1 = require("lodash");
const util_1 = require("./util");
const WAIT_TICK_INTERVAL = 10000; // in milliseconds
exports.DEFAULT_SCOPE = '_default';
exports.DEFAULT_COLLECTION = '_default';
function normalizeIndex(index) {
if (!index.bucket_id) {
// no bucket_id means we're not in a collection, let's normalize to the default collection
return {
...index,
bucket_id: index.keyspace_id,
scope_id: exports.DEFAULT_SCOPE,
keyspace_id: exports.DEFAULT_COLLECTION,
};
}
return index;
}
function normalizeStatus(status) {
// Older versions of Couchbase Server without scope/collection support won't return those values
// Add them if they are missing
return {
scope: exports.DEFAULT_SCOPE,
collection: exports.DEFAULT_COLLECTION,
...status
};
}
// Note: Assumes both the index and status have been normalized
function isStatusMatch(index, status) {
// Remove (replica X) from the end of the index name
const match = /^([^\s]*)/.exec(status.index);
const indexName = match ? match[1] : '';
return index.name === indexName &&
index.scope === status.scope &&
index.collection === status.collection;
}
function getKeyspace(bucket, scope = exports.DEFAULT_SCOPE, collection = exports.DEFAULT_COLLECTION) {
if (scope === exports.DEFAULT_SCOPE && collection === exports.DEFAULT_COLLECTION) {
return (0, util_1.ensureEscaped)(bucket);
}
else {
return `${(0, util_1.ensureEscaped)(bucket)}.${(0, util_1.ensureEscaped)(scope)}.${(0, util_1.ensureEscaped)(collection)}`;
}
}
exports.getKeyspace = getKeyspace;
/**
* Manages Couchbase indexes
*/
class IndexManager {
/**
* @param {Bucket} bucket
* @param {Cluster} cluster
*/
constructor(bucket, cluster) {
this.bucket = bucket;
this.cluster = cluster;
this.bucket = bucket;
this.cluster = cluster;
this.manager = cluster.queryIndexes();
}
/**
* Get the name of the bucket being managed
*/
get bucketName() {
return this.bucket.name;
}
/**
* Gets all indexes using a query
* Workaround until https://issues.couchbase.com/projects/JSCBC/issues/JSCBC-772 is resolved
*/
async getAllIndexes() {
let qs = '';
qs += 'SELECT idx.* FROM system:indexes AS idx';
qs += ` WHERE (bucket_id IS MISSING AND keyspace_id = "${this.bucketName}")`;
qs += ` OR bucket_id = "${this.bucketName}"`;
qs += ' AND `using`="gsi" ORDER BY is_primary DESC, name ASC';
const res = await this.cluster.query(qs);
return res.rows.map(normalizeIndex);
}
/**
* Gets index statuses for the bucket via the cluster manager
*/
async getIndexStatuses() {
const resp = await this.manager._http.request({
type: httpexecutor_1.HttpServiceType.Management,
method: httpexecutor_1.HttpMethod.Get,
path: '/indexStatus',
timeout: 5000,
});
const body = resp.body ? JSON.parse(resp.body.toString()) : null;
if (resp.statusCode !== 200) {
if (!body) {
throw new Error(`operation failed (${resp.statusCode})`);
}
throw new Error(body.reason);
}
const indexStatuses = body.indexes
.filter((index) => {
return index.bucket === this.bucketName;
})
.map(normalizeStatus);
return indexStatuses;
}
async getIndexes(scope, collection) {
const indexes = (await this.getAllIndexes())
.filter(index => !scope || (index.scope_id === scope && index.keyspace_id === collection))
.map(index => ({
// Enrich with additional fields. These will be further updated using status below.
...index,
scope: index.scope_id,
collection: index.keyspace_id,
nodes: [],
num_replica: -1,
num_partition: 1,
retain_deleted_xattr: false
}));
// Get additional info from the index status API, and map by name
const statuses = await this.getIndexStatuses();
// Apply hosts from index status API to index information
statuses.forEach((status) => {
const index = indexes.find((index) => isStatusMatch(index, status));
if (index) {
// add any hosts listed to index info
// but only for the first if partitioned (others will be dupes)
if (!index.partition || index.nodes.length === 0) {
index.nodes.push(...status.hosts);
}
// Each status record we find beyond the first indicates one replica
// We started at -1 so if there aren't any replicas we'll end at 0
index.num_replica++;
index.retain_deleted_xattr =
/"retain_deleted_xattr"\s*:\s*true/.test(status.definition);
index.num_partition = status.numPartition;
}
});
return indexes;
}
/**
* Creates an index based on an index definition
*/
async createIndex(statement) {
await this.cluster.query(statement, { adhoc: true });
}
getAlterStatement(indexName, scope, collection, withClause) {
let statement;
if (scope === exports.DEFAULT_SCOPE && collection === exports.DEFAULT_COLLECTION) {
// We need to use the old syntax for the default collection for backward compatibilty
statement = `ALTER INDEX ${(0, util_1.ensureEscaped)(this.bucketName)}.${(0, util_1.ensureEscaped)(indexName)} WITH `;
}
else {
statement = `ALTER INDEX ${(0, util_1.ensureEscaped)(indexName)} ON ${getKeyspace(this.bucketName, scope, collection)} WITH `;
}
statement += JSON.stringify(withClause);
return statement;
}
/**
* Moves index replicas been nodes
*/
async moveIndex(indexName, scope, collection, nodes) {
const withClause = {
action: 'move',
nodes: nodes,
};
const statement = this.getAlterStatement(indexName, scope, collection, withClause);
await this.cluster.query(statement, { adhoc: true });
}
/**
* Moves index replicas been nodes
*/
async resizeIndex(indexName, scope, collection, numReplica, nodes) {
const withClause = {
action: 'replica_count',
num_replica: numReplica,
};
if (nodes) {
withClause.nodes = nodes;
}
const statement = this.getAlterStatement(indexName, scope, collection, withClause);
console.log(statement);
await this.cluster.query(statement, { adhoc: true });
}
/**
* Builds any outstanding deferred indexes on the bucket
*/
async buildDeferredIndexes(scope = exports.DEFAULT_SCOPE, collection = exports.DEFAULT_COLLECTION) {
// Because the built-in buildDeferredIndexes doesn't filter by scope/collection, we must build ourselves
const deferredList = (await this.getAllIndexes()).filter(index => index.scope_id === scope &&
index.keyspace_id === collection &&
(index.state === 'deferred' || index.state === 'pending'))
.map(index => index.name);
// If there are no deferred indexes, we have nothing to do.
if (deferredList.length === 0) {
return [];
}
const keyspace = scope === exports.DEFAULT_SCOPE && collection === exports.DEFAULT_COLLECTION
? `\`${this.bucketName}\``
: `\`${this.bucketName}\`.\`${scope}\`.\`${collection}\``;
let qs = '';
qs += `BUILD INDEX ON ${keyspace} `;
qs += '(';
for (let j = 0; j < deferredList.length; ++j) {
if (j > 0) {
qs += ', ';
}
qs += '`' + deferredList[j] + '`';
}
qs += ')';
// Run our deferred build query
await this.cluster.query(qs);
return deferredList;
}
/**
* Monitors building indexes and triggers a Promise when complete
*/
async waitForIndexBuild(options, tickHandler) {
const effectiveOptions = {
scope: exports.DEFAULT_SCOPE,
collection: exports.DEFAULT_COLLECTION,
...options,
};
const startTime = Date.now();
let lastTick = startTime;
/** Internal to trigger the tick */
function testTick() {
const now = Date.now();
const interval = now - lastTick;
if (interval >= WAIT_TICK_INTERVAL) {
lastTick = now;
if (tickHandler) {
tickHandler(now - startTime);
}
}
}
while (!effectiveOptions.timeoutMs ||
(Date.now() - startTime < effectiveOptions.timeoutMs)) {
const indexes = (await this.getAllIndexes())
.filter(index => index.scope_id === effectiveOptions.scope &&
index.keyspace_id === effectiveOptions.collection &&
index.state !== 'online');
if (indexes.length === 0) {
// All indexes are online
return true;
}
// Because getIndexes has a latency,
// To get more accurate ticks check before and after the wait
testTick();
await new Promise((resolve) => setTimeout(resolve, 1000));
testTick();
}
// Timeout
return false;
}
/**
* Drops an existing index
*/
async dropIndex(indexName, scope = exports.DEFAULT_SCOPE, collection = exports.DEFAULT_COLLECTION, options) {
if (scope === exports.DEFAULT_SCOPE && collection === exports.DEFAULT_COLLECTION) {
await this.manager.dropIndex(this.bucketName, indexName, options);
}
else {
const qs = `DROP INDEX ${(0, util_1.ensureEscaped)(indexName)} ON ${getKeyspace(this.bucketName, scope, collection)}`;
// Run our deferred build query
await this.cluster.query(qs, {
timeout: options === null || options === void 0 ? void 0 : options.timeout,
adhoc: true
});
}
}
/**
* Gets the version of the cluster
*/
async getClusterVersion() {
const resp = await this.manager._http.request({
type: httpexecutor_1.HttpServiceType.Management,
method: httpexecutor_1.HttpMethod.Get,
path: '/pools/default',
timeout: 5000,
});
if (resp.statusCode !== 200) {
let errData = null;
try {
errData = JSON.parse(resp.body.toString());
}
catch (e) {
// ignore
}
if (!errData) {
throw new Error(`operation failed (${resp.statusCode})`);
}
throw new Error(errData.reason);
}
const poolData = JSON.parse(resp.body.toString());
const minCompatibility = poolData.nodes.reduce((accum, value) => {
if (value.clusterCompatibility < accum) {
accum = value.clusterCompatibility;
}
return accum;
}, 65535 * 65536);
const clusterCompatibility = minCompatibility < 65535 * 65536 ?
minCompatibility :
0;
return {
major: Math.floor(clusterCompatibility / 65536),
minor: clusterCompatibility & 65535,
};
}
/**
* Uses EXPLAIN to get a query plan for a statement
*/
async getQueryPlan(statement) {
statement = 'EXPLAIN ' + statement;
const explain = (await this.cluster.query(statement, { adhoc: true }));
const plan = explain.rows[0].plan;
if (plan && plan.keys) {
// Couchbase 5.0 started nesting within expr property
// so normalize the returned object
plan.keys = plan.keys.map((key) => (0, lodash_1.isString)(key) ? { expr: key } : key);
}
return plan;
}
}
exports.IndexManager = IndexManager;
//# sourceMappingURL=index-manager.js.map