reduct-js
Version:
ReductStore Client SDK for Javascript/NodeJS/Typescript
508 lines (507 loc) • 19.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Bucket = void 0;
const BucketSettings_1 = require("./messages/BucketSettings");
const BucketInfo_1 = require("./messages/BucketInfo");
const EntryInfo_1 = require("./messages/EntryInfo");
const Record_1 = require("./Record");
const APIError_1 = require("./APIError");
const stream_1 = __importStar(require("stream"));
const buffer_1 = require("buffer");
const Batch_1 = require("./Batch");
const Client_1 = require("./Client");
const QueryEntry_1 = require("./messages/QueryEntry");
/**
* Represents a bucket in ReductStore
*/
class Bucket {
/**
* Create a bucket. Use Client.creatBucket or Client.getBucket instead it
* @constructor
* @param name
* @param httpClient
* @see {Client}
*/
constructor(name, httpClient) {
this.name = name;
this.httpClient = httpClient;
this.isBrowser = typeof window !== "undefined";
this.readRecord = this.readRecord.bind(this);
}
/**
* Get bucket settings
* @async
* @return {Promise<BucketSettings>}
*/
async getSettings() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return Promise.resolve(BucketSettings_1.BucketSettings.parse(data.settings));
}
/**
* Set bucket settings
* @async
* @param settings {BucketSettings} new settings (you can set a part of settings)
*/
async setSettings(settings) {
await this.httpClient.put(`/b/${this.name}`, BucketSettings_1.BucketSettings.serialize(settings));
}
/**
* Get information about a bucket
* @async
* @return {Promise<BucketInfo>}
*/
async getInfo() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return BucketInfo_1.BucketInfo.parse(data.info);
}
/**
* Get entry list
* @async
* @return {Promise<EntryInfo>}
*/
async getEntryList() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return Promise.resolve(data.entries.map((entry) => EntryInfo_1.EntryInfo.parse(entry)));
}
/**
* Remove bucket
* @async
* @return {Promise<void>}
*/
async remove() {
await this.httpClient.delete(`/b/${this.name}`);
}
/**
* Remove an entry
* @async
* @param entry {string} name of the entry
* @return {Promise<void>}
*/
async removeEntry(entry) {
await this.httpClient.delete(`/b/${this.name}/${entry}`);
}
/**
* Remove a record
* @param entry {string} name of the entry
* @param ts {BigInt} timestamp of record in microseconds
*/
async removeRecord(entry, ts) {
await this.httpClient.delete(`/b/${this.name}/${entry}?ts=${ts}`);
}
/**
* Remove a batch of records
* @param entry {string} name of the entry
* @param tsList {BigInt[]} list of timestamps of records in microseconds
*/
async beginRemoveBatch(entry) {
return new Batch_1.Batch(this.name, entry, this.httpClient, Batch_1.BatchType.REMOVE);
}
/**
* Remove records by query
* @param entry {string} name of the entry
* @param start {BigInt} start point of the time period, if undefined, the query starts from the first record
* @param stop {BigInt} stop point of the time period. If undefined, the query stops at the last record
* @param options {QueryOptions} options for query. You can use only include, exclude, eachS, eachN other options are ignored
*/
async removeQuery(entry, start, stop, options) {
if (options !== undefined && options.when !== undefined) {
const { data } = await this.httpClient.post(`/b/${this.name}/${entry}/q`, QueryEntry_1.QueryOptions.serialize(QueryEntry_1.QueryType.REMOVE, options));
return Promise.resolve(data["removed_records"]);
}
else {
const ret = this.parse_query_params(start, stop, options);
const { data } = await this.httpClient.delete(`/b/${this.name}/${entry}/q?${ret.query}`);
return Promise.resolve(data["removed_records"]);
}
}
/**
* Start writing a record into an entry
* @param entry name of the entry
* @param options {BigInt | WriteOptions} timestamp in microseconds for the record or options. It is current time if undefined.
* @return Promise<WritableRecord>
* @example
* const record = await bucket.beginWrite("entry", {
* ts: 12345667n
* labels: {label1: "value1", label2: "value2"}
* contentType: "text/plain"
* );
* await record.write("Hello!");
*/
async beginWrite(entry, options) {
let localOptions = { ts: undefined };
if (options !== undefined) {
if (typeof options === "bigint") {
localOptions = { ts: options };
}
else {
localOptions = options;
}
}
localOptions.ts = localOptions.ts ?? BigInt(Date.now()) * 1000n;
return Promise.resolve(new Record_1.WritableRecord(this.name, entry, localOptions, this.httpClient));
}
/**
* Update labels of an existing record
*
* If a label has empty string value, it will be removed.
*
* @param entry {string} name of the entry
* @param ts {BigInt} timestamp of record in microseconds
* @param labels {LabelMap} labels to update
*/
async update(entry, ts, labels) {
const headers = {
"Content-Length": "0",
};
for (const [key, value] of Object.entries(labels)) {
headers[`x-reduct-label-${key}`] = value.toString();
}
await this.httpClient.patch(`/b/${this.name}/${entry}?ts=${ts}`, "", {
headers,
});
}
/**
* Start reading a record from an entry
* @param entry name of the entry
* @param ts {BigInt} timestamp of record in microseconds. Get the latest one, if undefined
* @param head {boolean} return only head of the record
* @return Promise<ReadableRecord>
*/
async beginRead(entry, ts, head) {
return await this.readRecord(entry, head ?? false, ts ? ts.toString() : undefined);
}
/**
* Rename an entry
* @param entry entry name to rename
* @param newEntry new entry name
*/
async renameEntry(entry, newEntry) {
await this.httpClient.put(`/b/${this.name}/${entry}/rename`, {
new_name: newEntry,
}, {
headers: {
"Content-Type": "application/json",
},
});
}
/**
* Rename a bucket
* @param newName new name of the bucket
*/
async rename(newName) {
await this.httpClient.put(`/b/${this.name}/rename`, {
new_name: newName,
}, {
headers: {
"Content-Type": "application/json",
},
});
this.name = newName;
}
/**
* Query records for a time interval as generator
* @param entry entry name
* @param entry {string} name of the entry
* @param start {BigInt} start point of the time period
* @param stop {BigInt} stop point of the time period
* @param options {number | QueryOptions} if number it is TTL of query on the server side, otherwise it is options for query
* @example
* for await (const record in bucket.query("entry-1", start, stop)) {
* console.log(record.ts, record.size);
* console.log(record.labels);
* const content = await record.read();
* // or use pipe
* const fileStream = fs.createWriteStream(`ts_${record.size}.txt`);
* record.pipe(fileStream);
* }
*/
async *query(entry, start, stop, options) {
let id;
let header_api_version;
let continuous = false;
let pollInterval = 1;
let head = false;
if (options !== undefined &&
typeof options === "object" &&
"when" in options) {
const { data, headers } = await this.httpClient.post(`/b/${this.name}/${entry}/q`, QueryEntry_1.QueryOptions.serialize(QueryEntry_1.QueryType.QUERY, options));
({ id } = data);
header_api_version = headers["x-reduct-api"];
continuous = options.continuous ?? false;
pollInterval = options.pollInterval ?? 1;
head = options.head ?? false;
}
else {
// TODO: remove this block after 1.xx
const ret = this.parse_query_params(start, stop, options);
const url = `/b/${this.name}/${entry}/q?` + ret.query;
const { data, headers } = await this.httpClient.get(url);
({ id } = data);
header_api_version = headers["x-reduct-api"];
({ continuous, pollInterval, head } = ret);
}
if ((0, Client_1.isCompatibale)("1.5", header_api_version) && !this.isBrowser) {
yield* this.fetchAndParseBatchedRecords(entry, id, continuous, pollInterval, head);
}
else {
yield* this.fetchAndParseSingleRecord(entry, id, continuous, pollInterval, head);
}
}
getName() {
return this.name;
}
parse_query_params(start, stop, options) {
let continueQuery = false;
let poolInterval = 1;
const params = [];
let head = false;
if (start !== undefined) {
params.push(`start=${start}`);
}
if (stop !== undefined) {
params.push(`stop=${stop}`);
}
if (options !== undefined) {
if (typeof options === "number") {
params.push(`ttl=${options}`);
}
else {
// Build query string from options
if (options.ttl !== undefined) {
params.push(`ttl=${options.ttl}`);
}
for (const [key, value] of Object.entries(options.include ? options.include : {})) {
params.push(`include-${key}=${value}`);
}
for (const [key, value] of Object.entries(options.exclude ? options.exclude : {})) {
params.push(`exclude-${key}=${value}`);
}
if (options.eachS !== undefined) {
params.push(`each_s=${options.eachS}`);
}
if (options.eachN !== undefined) {
params.push(`each_n=${options.eachN}`);
}
if (options.limit !== undefined) {
params.push(`limit=${options.limit}`);
}
if (options.continuous !== undefined) {
params.push(`continuous=${options.continuous ? "true" : "false"}`);
continueQuery = options.continuous;
if (options.pollInterval !== undefined) {
// eslint-disable-next-line prefer-destructuring
poolInterval = options.pollInterval;
}
// Set default TTL for continue query as 2 * poolInterval
if (options.ttl === undefined) {
params.push(`ttl=${poolInterval * 2}`);
}
}
continueQuery = options.continuous ?? false;
poolInterval = options.pollInterval ?? 1;
head = options.head ?? false;
}
}
return {
continuous: continueQuery,
pollInterval: poolInterval,
head: head,
query: params.join("&"),
};
}
async *fetchAndParseSingleRecord(entry, id, continueQuery, pollInterval, head) {
while (true) {
try {
const record = await this.readRecord(entry, head, undefined, id);
yield record;
}
catch (e) {
if (e instanceof APIError_1.APIError && e.status === 204) {
if (continueQuery) {
await new Promise((resolve) => setTimeout(resolve, pollInterval * 1000));
continue;
}
return;
}
throw e;
}
}
}
async readRecord(entry, head, ts, id) {
let param = "";
if (ts !== undefined) {
param = `ts=${ts}`;
}
if (id !== undefined) {
param = `q=${id}`;
}
const request = head ? this.httpClient.head : this.httpClient.get;
const { status, headers, data } = await request(`/b/${this.name}/${entry}?${param}`, head
? undefined
: {
responseType: this.isBrowser ? "arraybuffer" : "stream",
});
if (status === 204) {
throw new APIError_1.APIError(headers["x-reduct-error"] ?? "No content", 204);
}
const labels = {};
for (const [key, value] of Object.entries(headers)) {
if (key.startsWith("x-reduct-label-")) {
labels[key.substring(15)] = value;
}
}
if (this.isBrowser) {
// Pass a dummy Stream object and use ArrayBuffer
const arrayBuffer = data;
return new Record_1.ReadableRecord(BigInt(headers["x-reduct-time"] ?? 0), BigInt(headers["content-length"] ?? 0), headers["x-reduct-last"] == "1", head, new stream_1.default.Readable(), labels, headers["content-type"] ?? "application/octet-stream", arrayBuffer);
}
else {
// Pass the actual Stream object to ReadableRecord
const stream = data;
return new Record_1.ReadableRecord(BigInt(headers["x-reduct-time"] ?? 0), BigInt(headers["content-length"] ?? 0), headers["x-reduct-last"] == "1", head, stream, labels, headers["content-type"] ?? "application/octet-stream");
}
}
async *fetchAndParseBatchedRecords(entry, id, continueQuery, poolInterval, head) {
while (true) {
try {
for await (const record of this.readBatchedRecords(entry, head, id)) {
yield record;
if (record.last) {
return;
}
}
}
catch (e) {
if (e instanceof APIError_1.APIError && e.status === 204) {
if (continueQuery) {
await new Promise((resolve) => setTimeout(resolve, poolInterval * 1000));
continue;
}
return;
}
throw e;
}
}
}
async *readBatchedRecords(entry, head, id) {
const request = head ? this.httpClient.head : this.httpClient.get;
const { status, headers, data } = await request(`/b/${this.name}/${entry}/batch?q=${id}`, head ? undefined : { responseType: "stream" });
if (status === 204) {
throw new APIError_1.APIError(headers["x-reduct-error"] ?? "No content", 204);
}
let count = 0;
const total = Object.entries(headers).reduce((acc, [key, _]) => (key.startsWith("x-reduct-time-") ? acc + 1 : acc), 0);
let last = false;
for (const [key, value] of Object.entries(headers)) {
if (!key.startsWith("x-reduct-time-"))
continue;
const ts = key.substring(14);
const { size, contentType, labels } = parseCsvRow(value);
count += 1;
let stream;
if (count === total) {
if (headers["x-reduct-last"] === "true") {
last = true;
}
stream = data;
}
else {
let buffer = buffer_1.Buffer.from([]);
if (!head) {
buffer = await new Promise((resolve, reject) => {
const err_handler = (err) => {
reject(err);
};
data.on("readable", function handler() {
const chunk = data.read(Number(size));
if (chunk !== null) {
resolve(chunk);
data.off("readable", handler);
data.off("error", err_handler);
}
});
data.on("error", err_handler);
});
}
stream = stream_1.Readable.from(buffer);
}
yield new Record_1.ReadableRecord(BigInt(ts), size, last, head, stream, labels, contentType);
}
}
/**
* Create a new batch for writing records to the database.
* @param entry
*/
async beginWriteBatch(entry) {
return new Batch_1.Batch(this.name, entry, this.httpClient, Batch_1.BatchType.WRITE);
}
/**
* Create a new batch for updating records in the database.
* @param entry
*/
async beginUpdateBatch(entry) {
return new Batch_1.Batch(this.name, entry, this.httpClient, Batch_1.BatchType.UPDATE);
}
}
exports.Bucket = Bucket;
function parseCsvRow(row) {
const items = [];
let escaped = "";
for (const item of row.split(",")) {
if (item.startsWith('"') && !escaped) {
escaped = item.substring(1);
}
if (escaped) {
if (item.endsWith('"')) {
escaped = escaped.slice(0, -1);
items.push(escaped);
escaped = "";
}
else {
escaped += item;
}
}
else {
items.push(item);
}
}
const size = BigInt(items[0]);
// eslint-disable-next-line prefer-destructuring
const contentType = items[1];
const labels = {};
for (const item of items.slice(2)) {
if (!item.includes("=")) {
continue;
}
const [key, value] = item.split("=", 2);
labels[key] = value;
}
return {
size,
contentType,
labels,
};
}