couchbase-index-manager
Version:
Manage Couchbase indexes during the CI/CD process
425 lines • 15.8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.IndexDefinition = void 0;
const tslib_1 = require("tslib");
const lodash_1 = (0, tslib_1.__importDefault)(require("lodash"));
const configuration_1 = require("../configuration");
const feature_versions_1 = require("../feature-versions");
const index_manager_1 = require("../index-manager");
const plan_1 = require("../plan");
const util_1 = require("../util");
const index_definition_base_1 = require("./index-definition-base");
/**
* Ensures that a server name has a port number appended, defaults to 8091
*/
function ensurePort(server) {
if (/:\d+$/.exec(server)) {
return server;
}
else {
return server + ':8091';
}
}
function processKey(index, key, processor, initialValue) {
const result = processor.call(index, initialValue);
if (result !== undefined) {
index[key] = result;
}
}
/**
* Map of processing functions to handle hash keys.
* "this" when the function is called will be the IndexDefinition.
* If a value is returned, it is assigned to the key.
* If "undefined" is returned, it assumed that the handler
* processed the value completely.
*/
const keys = {
is_primary: (val) => !!val,
index_key: (val) => !val ? [] :
lodash_1.default.isString(val) ?
lodash_1.default.compact([val]) :
Array.from(val),
condition: (val) => { var _a; return (_a = val) !== null && _a !== void 0 ? _a : ''; },
partition: function (val) {
// For overrides, ignore undefined
// But clear the entire value if null
if (!lodash_1.default.isUndefined(val)) {
if (!val) {
this.partition = undefined;
}
else {
this.partition = {
...this.partition,
...val
};
}
}
return undefined;
},
nodes: function (val) {
this.nodes = val;
// for partitioned index, num_replica and nodes
// are decoupled so skip setting num_replica
if (val && val.length && !this.partition) {
this.num_replica = val.length - 1;
}
return undefined;
},
manual_replica: (val) => !!val,
num_replica: function (val) {
var _a, _b;
if (!this.partition) {
return (_a = val) !== null && _a !== void 0 ? _a : (this.nodes ? this.nodes.length - 1 : 0);
}
else {
// for partitioned index, num_replica and nodes
// are decoupled so skip nodes check
return (_b = val) !== null && _b !== void 0 ? _b : 0;
}
},
retain_deleted_xattr: (val) => !!val,
lifecycle: function (val) {
if (!this.lifecycle) {
this.lifecycle = {};
}
if (val) {
lodash_1.default.extend(this.lifecycle, val);
}
return undefined;
},
post_process: function (val) {
let fn = null;
if (lodash_1.default.isFunction(val)) {
fn = val;
}
else if (lodash_1.default.isString(val)) {
// eslint-disable-next-line @typescript-eslint/no-implied-eval
fn = new Function('require', 'process', val);
}
fn === null || fn === void 0 ? void 0 : fn.call(this, require, process);
return undefined;
},
};
/**
* Represents an index
*/
class IndexDefinition extends index_definition_base_1.IndexDefinitionBase {
/**
* Creates a new IndexDefinition from a simple object map
*/
constructor(configuration) {
super(configuration);
this.applyOverride(configuration, true);
}
/**
* Creates a new IndexDefinition from a simple object map
*/
static fromObject(configuration) {
return new IndexDefinition(configuration);
}
/**
* Apply overrides to the index definition
*/
applyOverride(override, applyMissing) {
var _a;
// Process the keys
let key;
for (key in keys) {
if (applyMissing || override[key] !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
processKey(this, key, keys[key], override[key]);
}
}
// Validate the resulting defintion
(_a = configuration_1.IndexValidators.post_validate) === null || _a === void 0 ? void 0 : _a.call(this);
}
/**
* Gets the required index mutations, if any, to sync this definition
*/
*getMutations(context) {
this.normalizeNodeList(context.currentIndexes);
const mutations = [];
if (!this.manual_replica) {
mutations.push(...this.getMutation(context));
}
else {
for (let i = 0; i <= this.num_replica; i++) {
mutations.push(...this.getMutation(context, i));
}
if (!this.is_primary) {
// Handle dropping replicas if the count is lowered
for (let i = this.num_replica + 1; i <= 10; i++) {
mutations.push(...this.getMutation(context, i, true));
}
}
}
IndexDefinition.phaseMutations(mutations);
yield* mutations;
}
*getMutation(context, replicaNum = 0, forceDrop = false) {
var _a;
const suffix = !replicaNum ?
'' :
`_replica${replicaNum}`;
const currentIndex = context.currentIndexes.find((index) => {
return this.isMatch(index, suffix);
});
const drop = forceDrop || ((_a = this.lifecycle) === null || _a === void 0 ? void 0 : _a.drop);
if (!currentIndex) {
// Index isn't found
if (!drop) {
yield new plan_1.CreateIndexMutation(this, this.name + suffix, this.getWithClause(replicaNum));
}
}
else if (drop) {
yield new plan_1.DropIndexMutation(this, currentIndex.name);
}
else if (!this.is_primary && this.requiresUpdate(currentIndex)) {
yield new plan_1.UpdateIndexMutation(this, this.name + suffix, this.getWithClause(replicaNum), currentIndex);
}
else if (!this.manual_replica &&
!lodash_1.default.isUndefined(currentIndex.num_replica) &&
this.num_replica !== currentIndex.num_replica) {
// Number of replicas changed for an auto replica index
// We must drop and recreate.
if (feature_versions_1.FeatureVersions.alterIndexReplicaCount(context.clusterVersion)) {
yield new plan_1.ResizeIndexMutation(this, this.name + suffix);
}
else {
yield new plan_1.UpdateIndexMutation(this, this.name + suffix, this.getWithClause(replicaNum), currentIndex);
}
}
else if (this.nodes && currentIndex.nodes) {
// Check for required node changes
currentIndex.nodes.sort();
if (this.manual_replica) {
if (this.nodes[replicaNum] !== currentIndex.nodes[0]) {
yield new plan_1.UpdateIndexMutation(this, this.name + suffix, this.getWithClause(replicaNum), currentIndex);
}
}
else {
if (!lodash_1.default.isEqual(this.nodes, currentIndex.nodes)) {
yield new plan_1.MoveIndexMutation(this, this.name + suffix);
}
}
}
}
getWithClause(replicaNum) {
let withClause;
if (!this.manual_replica) {
withClause = {
nodes: this.nodes ? this.nodes.map(ensurePort) : undefined,
num_replica: this.num_replica,
};
}
else {
withClause = {
nodes: this.nodes && [ensurePort(this.nodes[replicaNum !== null && replicaNum !== void 0 ? replicaNum : 0])],
};
}
if (this.retain_deleted_xattr) {
withClause = {
...withClause,
retain_deleted_xattr: true,
};
}
if (this.partition && this.partition.num_partition) {
withClause = {
...withClause,
num_partition: this.partition.num_partition,
};
}
return withClause;
}
/**
* Formats the PartitionHash as a string
*/
getPartitionString() {
if (!this.partition) {
return '';
}
let str = `${(this.partition.strategy || configuration_1.PartitionStrategy.Hash).toUpperCase()}(`;
str += this.partition.exprs.join();
str += ')';
return str;
}
/**
* Tests to see if a Couchbase index matches this definition
*/
isMatch(index, suffix) {
// First validate we're in the correct collection
if (index.scope !== this.scope || index.collection !== this.collection) {
return false;
}
// Then validate the name
if (this.is_primary) {
// Consider any primary index a match, regardless of name
return !!index.is_primary;
}
else {
return (0, util_1.ensureEscaped)(this.name + (suffix || '')) === (0, util_1.ensureEscaped)(index.name);
}
}
/**
* Tests to see if a Couchbase index requires updating,
* ignoring node changes which are handled separately.
*/
requiresUpdate(index) {
return (index.condition || '') !== this.condition ||
!lodash_1.default.isEqual(index.index_key, this.index_key) ||
(index.partition || '') !== this.getPartitionString() ||
(this.partition && this.partition.num_partition &&
this.partition.num_partition !== index.num_partition) ||
!!index.retain_deleted_xattr !== this.retain_deleted_xattr;
}
/**
* Normalizes the index definition using Couchbase standards
* for condition and index_key.
*/
async normalize(manager) {
if (this.is_primary || (this.lifecycle && this.lifecycle.drop)) {
// Not required for primary index or drops
return;
}
// Calling explain for creating an index returns a plan
// in which the keys and condition have been normalizaed for us
// However, we must use a special index name to prevent rejection
// due to name conflicts.
const statement = this.getCreateStatement(manager.bucketName, '__cbim_normalize');
let plan;
try {
plan = await manager.getQueryPlan(statement);
}
catch (e) {
throw new Error(`Invalid index definition for ${this.name}: ${e.message}`);
}
this.index_key = (plan.keys || []).map((key) => key.expr + (key.desc ? ' DESC' : ''));
this.condition = plan.where || '';
if (plan.partition) {
this.partition = {
...plan.partition,
num_partition: this.partition ? this.partition.num_partition : undefined,
};
}
else {
this.partition = undefined;
}
}
getCreateStatement(bucketName, indexNameOrWithClause, withClause) {
let indexName;
if (!lodash_1.default.isString(indexNameOrWithClause)) {
withClause = indexNameOrWithClause;
}
else {
indexName = indexNameOrWithClause;
}
indexName = (0, util_1.ensureEscaped)(indexName || this.name);
const keyspace = (0, index_manager_1.getKeyspace)(bucketName, this.scope, this.collection);
let statement;
if (this.is_primary) {
statement = `CREATE PRIMARY INDEX ${indexName}`;
statement += ` ON ${keyspace}`;
}
else {
statement = `CREATE INDEX ${indexName}`;
statement += ` ON ${keyspace}`;
statement += ` (${this.index_key.join(', ')})`;
}
if (this.partition) {
statement +=
` PARTITION BY ${this.getPartitionString()}`;
}
if (!this.is_primary) {
if (this.condition) {
statement += ` WHERE ${this.condition}`;
}
}
withClause = lodash_1.default.extend({}, withClause, {
defer_build: true,
});
if (!withClause.num_replica) {
// Don't include in the query string if not > 0
delete withClause.num_replica;
}
if (!withClause.nodes || withClause.nodes.length === 0) {
// Don't include an empty value
delete withClause.nodes;
}
statement += ' WITH ' + JSON.stringify(withClause);
return statement;
}
/**
* Apply phases to the collection of index mutations
*/
static phaseMutations(mutations) {
// All creates should be in phase one
// All updates should be in one phase each, after creates
// Everything else should be in the last phase
// This is relative to each index definition only
let nextPhase = 1;
for (const mutation of mutations) {
if (mutation instanceof plan_1.CreateIndexMutation) {
nextPhase = 2;
mutation.phase = 1;
}
}
for (const mutation of mutations) {
if (mutation instanceof plan_1.UpdateIndexMutation) {
mutation.phase = nextPhase;
nextPhase += 1;
}
}
for (const mutation of mutations) {
if (!(mutation instanceof plan_1.CreateIndexMutation) &&
!(mutation instanceof plan_1.UpdateIndexMutation)) {
mutation.phase = nextPhase;
}
}
}
/**
* Ensures that the node list has port numbers and is sorted in the same
* order as the current indexes. This allows easy matching of existing
* node assignments, reducing reindex load due to minor node shifts.
*/
normalizeNodeList(currentIndexes) {
if (!this.nodes) {
return;
}
this.nodes = this.nodes.map(ensurePort);
this.nodes.sort();
if (this.manual_replica) {
// We only care about specific node mappings for manual replicas
// For auto replicas we let Couchbase handle it
const newNodeList = [];
const unused = lodash_1.default.clone(this.nodes);
for (let replicaNum = 0; replicaNum <= this.num_replica; replicaNum++) {
const suffix = !replicaNum ?
'' :
`_replica${replicaNum}`;
const index = currentIndexes.find((index) => {
return this.isMatch(index, suffix);
});
if (index && index.nodes) {
const unusedIndex = unused.findIndex((name) => name === index.nodes[0]);
if (unusedIndex >= 0) {
newNodeList[replicaNum] =
unused.splice(unusedIndex, 1)[0];
}
}
}
// Fill in the remaining nodes that didn't have a match
for (let replicaNum = 0; replicaNum <= this.num_replica; replicaNum++) {
if (!newNodeList[replicaNum]) {
const nextUnused = unused.shift();
if (!nextUnused) {
throw new Error('Should be unreachable');
}
newNodeList[replicaNum] = nextUnused;
}
}
this.nodes = newNodeList;
}
}
}
exports.IndexDefinition = IndexDefinition;
//# sourceMappingURL=index-definition.js.map