@darlean/tables-suite
Version:
Tables Suite that provides fast, indexed access to structured data
440 lines (439 loc) • 19.3 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.TableActor = void 0;
const base_1 = require("@darlean/base");
const utils_1 = require("@darlean/utils");
const crypto = __importStar(require("crypto"));
class TableActor {
constructor(persistence, deser, id, shard) {
this.persistence = persistence;
this.deser = deser;
this.id = id;
this.internalId = [id.length.toString(), ...id];
this.shard = shard;
}
async put(request) {
const sortKey = ['base', ...request.id];
const isDelete = request.data === undefined;
const indexes = request.indexes;
const baseline = request.baseline
? this.decodeBaseline(request.baseline)
: await this.fetchBaseline(sortKey, request.specifier);
const hashes = new Map();
const newBaseline = {
indexes: []
};
const itemKey = JSON.stringify(request.id);
const batch = { items: [] };
if (!isDelete) {
// Persist new or changed index values
for (const index of indexes) {
const key = JSON.stringify([index.name, ...index.keys]);
const hash = this.calculateHash(index);
hashes.set(key, hash);
const bli = baseline?.indexes.find((x) => JSON.stringify([x.name, ...x.keys]) === key);
if (!bli || bli.hash !== hash) {
const entry = {
id: request.id,
data: this.deser.serialize(index.data)
};
batch.items.push({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKey: ['index', index.name, ...index.keys, itemKey, hash],
specifier: request.specifier,
value: this.deser.serialize(entry),
version: request.version,
identifier: ''
});
newBaseline.indexes.push({
name: index.name,
keys: index.keys,
hash
});
}
}
}
// Persist the actual data.
const baseItem = {
data: this.deser.serialize(request.data),
baseline: newBaseline
};
batch.items.push({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKey,
specifier: request.specifier,
value: isDelete ? undefined : this.deser.serialize(baseItem),
version: request.version,
identifier: ''
});
// Remove old index values.
for (const bli of baseline?.indexes ?? []) {
const key = JSON.stringify([bli.name, ...bli.keys]);
const index = indexes.filter((x) => JSON.stringify([x.name, ...x.keys]) === key);
const hash = hashes.get(key);
if (!index || hash !== bli.hash) {
batch.items.push({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKey: ['index', bli.name, ...bli.keys, itemKey, bli.hash ?? ''],
specifier: request.specifier,
value: undefined,
version: request.version,
identifier: ''
});
}
}
const results = await this.persistence.storeBatch(batch);
if (results.unprocessedItems.length > 0) {
throw new base_1.ApplicationError(base_1.APPLICATION_ERROR_TABLE_ERROR, '');
}
return {
baseline: isDelete ? undefined : this.encodeBaseline(newBaseline)
};
}
get(request) {
return this.getImpl(request);
}
search(request) {
return this.searchImpl(request);
}
async searchBuffer(request) {
return this.deser.serialize(await this.searchImpl(request));
}
async searchImpl(request) {
let operator = 'none';
const sortKey = [];
let sortKey2 = [];
const filterParts = [];
let phase = 'sortkey';
let idx = -1;
for (const c of request.keys ?? []) {
idx++;
if (phase === 'sortkey') {
sortKey.push(c.value);
switch (c.operator) {
case 'prefix':
operator = 'prefix';
phase = 'filter';
break;
case 'eq':
operator = 'exact';
break;
case 'gte':
operator = 'gte';
phase = 'filter';
break;
case 'lte':
operator = 'lte';
phase = 'filter';
break;
case 'between':
operator = 'between';
sortKey2 = [...sortKey.slice(0, -1), c.value2];
phase = 'filter';
break;
case 'contains':
phase = 'filter';
filterParts.push((0, base_1.contains)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
case 'containsni':
phase = 'filter';
filterParts.push((0, base_1.containsni)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
}
}
else {
switch (c.operator) {
case 'prefix':
filterParts.push((0, base_1.prefix)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
case 'eq':
filterParts.push((0, base_1.eq)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
case 'gte':
filterParts.push((0, base_1.gte)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
case 'lte':
filterParts.push((0, base_1.lte)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
case 'between':
filterParts.push((0, base_1.gte)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
filterParts.push((0, base_1.lte)((0, base_1.sk)(idx), (0, base_1.literal)(c.value2)));
break;
case 'contains':
filterParts.push((0, base_1.contains)((0, base_1.sk)(idx), (0, base_1.literal)(c.value)));
break;
}
}
}
if (request.filter) {
filterParts.push(request.filter.expression);
}
if (request.index) {
// Search of index table (not on base table)
const filter = filterParts.length > 0 ? (0, base_1.and)(...filterParts) : undefined;
const projection = request.indexProjection ? this.enhanceProjection(request.indexProjection) : undefined;
const [sortKeyFrom, sortKeyTo, sortKeyToMatch] = this.deriveKeyInfo(operator, sortKey, sortKey2);
const result = await this.persistence.query({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKeyFrom: this.prefixSortKey(['index', request.index], sortKeyFrom),
sortKeyTo: this.prefixSortKey(['index', request.index], sortKeyTo),
sortKeyToMatch,
sortKeyOrder: request.keysOrder ?? 'ascending',
specifier: request.specifier,
filterExpression: filter,
filterFieldBase: 'data',
filterSortKeyOffset: 2,
projectionFilter: projection,
continuationToken: request.continuationToken,
maxItems: request.maxChunkItems ?? request.maxItems
});
const response = {
items: [],
continuationToken: result.continuationToken
};
for (const item of result.items) {
if (item.value) {
const value = this.deser.deserialize(item.value);
const resultItem = {
id: value.id,
// Remove 'index' + name at beginning and itemkey+hash that are added at the end
keys: item.sortKey.slice(2, -2),
indexFields: request.indexRepresentation !== 'buffer' ? this.extractDataFields(value.data) : undefined,
indexBuffer: request.indexRepresentation === 'buffer' ? this.extractDataBuffer(value.data) : undefined
};
response.items.push(resultItem);
}
}
if (request.tableProjection) {
const tasks = [];
for (const item of response.items) {
const id = item.id;
tasks.push(() => {
return this.getImpl({
keys: id,
projection: request.tableProjection ? this.enhanceProjection(request.tableProjection) : undefined,
specifier: request.specifier,
representation: request.tableRepresentation
});
});
}
const tableResults = await (0, utils_1.parallel)(tasks, 5 * 1000, 100);
let idx = -1;
for (const result of tableResults.results) {
idx++;
if (result.result) {
if (request.tableRepresentation === 'buffer') {
response.items[idx].tableBuffer = result.result.dataBuffer;
}
else {
response.items[idx].tableFields = result.result.data;
}
}
}
}
return response;
}
else {
// Search on 'base' table data (not on an index)
const filter = filterParts.length > 0 ? (0, base_1.and)(...filterParts) : undefined;
const isEmptyProjection = this.isEmptyProjection(request.tableProjection);
const projection = isEmptyProjection
? request.tableProjection
: request.tableProjection
? this.enhanceProjection(request.tableProjection)
: undefined;
const [sortKeyFrom, sortKeyTo, sortKeyToMatch] = this.deriveKeyInfo(operator, sortKey, sortKey2);
const result = await this.persistence.query({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKeyFrom: this.prefixSortKey(['base'], sortKeyFrom),
sortKeyTo: this.prefixSortKey(['base'], sortKeyTo),
sortKeyToMatch,
sortKeyOrder: request.keysOrder ?? 'ascending',
specifier: request.specifier,
filterExpression: filter,
filterFieldBase: 'data',
filterSortKeyOffset: 1,
projectionFilter: projection,
continuationToken: request.continuationToken,
maxItems: request.maxChunkItems ?? request.maxItems
});
const response = {
items: [],
continuationToken: result.continuationToken
};
for (const item of result.items) {
if (isEmptyProjection) {
const resultItem = {
id: item.sortKey.slice(1)
};
response.items.push(resultItem);
}
else if (item.value) {
const value = this.deser.deserialize(item.value);
const resultItem = {
id: item.sortKey.slice(1),
tableFields: request.tableRepresentation !== 'buffer' ? this.extractDataFields(value.data) : undefined,
tableBuffer: request.tableRepresentation === 'buffer' ? this.extractDataBuffer(value.data) : undefined
};
response.items.push(resultItem);
}
}
return response;
}
}
extractDataFields(data) {
if (Buffer.isBuffer(data)) {
return this.deser.deserialize(data);
}
return data;
}
extractDataBuffer(data) {
if (data === undefined) {
return undefined;
}
if (Buffer.isBuffer(data)) {
return data;
}
return this.deser.serialize(data);
}
deriveKeyInfo(operator, sortKey, sortKey2) {
let sortKeyFrom;
let sortKeyTo;
let sortKeyLoose = false;
switch (operator) {
case 'exact':
sortKeyFrom = sortKey;
sortKeyTo = sortKey;
break;
case 'between':
sortKeyFrom = sortKey;
sortKeyTo = sortKey2;
break;
case 'gte':
sortKeyFrom = sortKey;
sortKeyTo = (sortKey?.length ?? 0) > 0 ? sortKey?.slice(0, sortKey.length - 1) : undefined;
break;
case 'lte':
sortKeyTo = sortKey;
sortKeyFrom = (sortKey?.length ?? 0) > 0 ? sortKey?.slice(0, sortKey.length - 1) : undefined;
break;
case 'prefix':
sortKeyFrom = sortKey;
sortKeyTo = sortKey;
sortKeyLoose = true;
break;
}
return [sortKeyFrom, sortKeyTo, sortKeyLoose ? 'loose' : 'strict'];
}
prefixSortKey(prefix, key) {
return [...prefix, ...(key ?? [])];
}
async fetchBaseline(keys, specifier) {
const result = await this.persistence.load({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKey: ['base', ...keys],
specifier: specifier,
projectionFilter: this.enhanceProjection([])
});
if (result.value) {
const baseItem = this.deser.deserialize(result.value);
return baseItem.baseline;
}
else {
return { indexes: [] };
}
}
async getImpl(request) {
const result = await this.persistence.load({
partitionKey: ['Table', ...this.internalId, this.shard.toString()],
sortKey: ['base', ...request.keys],
specifier: request.specifier,
projectionFilter: request.projection ? this.enhanceProjection(request.projection) : undefined,
projectionBases: ['data']
});
if (result.value) {
const baseItem = this.deser.deserialize(result.value);
return {
version: result.version ?? '',
data: request.representation !== 'buffer' ? this.extractDataFields(baseItem.data) : undefined,
dataBuffer: request.representation === 'buffer' ? this.extractDataBuffer(baseItem.data) : undefined,
baseline: this.encodeBaseline(baseItem.baseline)
};
}
else {
return {
version: result.version ?? '',
data: undefined,
baseline: this.encodeBaseline({ indexes: [] })
};
}
}
calculateHash(item) {
const hash = crypto.createHash('sha1');
for (const key of item.keys) {
hash.update(key);
hash.update('*');
}
if (item.data) {
hash.update(JSON.stringify(item.data));
}
return hash.digest('base64');
}
encodeBaseline(baseline) {
return Buffer.from(JSON.stringify(baseline), 'utf-8').toString('base64');
}
decodeBaseline(baseline) {
return baseline ? JSON.parse(Buffer.from(baseline, 'base64').toString('utf-8')) : undefined;
}
enhanceProjection(filter) {
const result = filter.map((x) => x[0] + 'data.' + x.substring(1));
result.push('-data.*');
result.push('+*');
return result;
}
isEmptyProjection(filter) {
return filter?.length === 1 && filter[0] === '-*';
}
}
__decorate([
(0, base_1.action)({ locking: 'shared' })
], TableActor.prototype, "put", null);
__decorate([
(0, base_1.action)({ locking: 'shared' })
], TableActor.prototype, "get", null);
__decorate([
(0, base_1.action)({ locking: 'shared' })
], TableActor.prototype, "search", null);
__decorate([
(0, base_1.action)({ locking: 'shared' })
], TableActor.prototype, "searchBuffer", null);
exports.TableActor = TableActor;