reduct-js
Version:
ReductStore Client SDK for Javascript/NodeJS/Typescript
394 lines (393 loc) • 15.4 kB
JavaScript
import { BucketSettings } from "./messages/BucketSettings.js";
import { APIError } from "./APIError.js";
import { BucketInfo } from "./messages/BucketInfo.js";
import { EntryInfo } from "./messages/EntryInfo.js";
import { ReadableRecord, WritableRecord } from "./Record.js";
import { Batch, BatchType } from "./Batch.js";
import { RecordBatch, RecordBatchType } from "./RecordBatch.js";
import { QueryOptions, QueryType } from "./messages/QueryEntry.js";
import { QueryLinkOptions } from "./messages/QueryLink.js";
import { fetchAndParseBatchV1 } from "./batch/BatchV1.js";
import { fetchAndParseBatchV2 } from "./batch/BatchV2.js";
//#region src/Bucket.ts
/**
* Represents a bucket in ReductStore
*/
var Bucket = class {
/**
* Create a bucket. Use Client.creatBucket or Client.getBucket instead it
* @constructor
* @param name
* @param httpClient
* @see {Client}
*/
constructor(name, httpClient) {
this.name = name;
this.httpClient = httpClient;
this.readRecord = this.readRecord.bind(this);
}
/**
* Get bucket settings
* @async
* @return {Promise<BucketSettings>}
*/
async getSettings() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return Promise.resolve(BucketSettings.parse(data.settings));
}
/**
* Set bucket settings
* @async
* @param settings {BucketSettings} new settings (you can set a part of settings)
*/
async setSettings(settings) {
await this.httpClient.put(`/b/${this.name}`, BucketSettings.serialize(settings));
}
/**
* Get information about a bucket
* @async
* @return {Promise<BucketInfo>}
*/
async getInfo() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return BucketInfo.parse(data.info);
}
/**
* Get entry list
* @async
* @return {Promise<EntryInfo>}
*/
async getEntryList() {
const { data } = await this.httpClient.get(`/b/${this.name}`);
return Promise.resolve(data.entries.map((entry) => EntryInfo.parse(entry)));
}
/**
* Remove bucket
* @async
* @return {Promise<void>}
*/
async remove() {
await this.httpClient.delete(`/b/${this.name}`);
}
/**
* Remove an entry
* @async
* @param entry {string} name of the entry
* @return {Promise<void>}
*/
async removeEntry(entry) {
await this.httpClient.delete(`/b/${this.name}/${entry}`);
}
/**
* Remove a record
* @param entry {string} name of the entry
* @param ts {BigInt} timestamp of record in microseconds
*/
async removeRecord(entry, ts) {
await this.httpClient.delete(`/b/${this.name}/${entry}?ts=${ts}`);
}
/**
* Remove a batch of records
* @param entry {string} name of the entry
* @param tsList {BigInt[]} list of timestamps of records in microseconds
*/
async beginRemoveBatch(entry) {
return new Batch(this.name, entry, this.httpClient, BatchType.REMOVE);
}
/**
* Remove records by query
* @param entry {string | string[]} name of the entry or entries
* @param start {BigInt} start point of the time period, if undefined, the query starts from the first record
* @param stop {BigInt} stop point of the time period. If undefined, the query stops at the last record
* @param options {QueryOptions} options for query. You can use only include, exclude, eachS, eachN other options are ignored
*/
async removeQuery(entry, start, stop, options) {
const entries = Array.isArray(entry) ? entry : [entry];
const localOptions = options ?? {};
let queryUrl = `/b/${this.name}/${entry}/q`;
let queryBody = QueryOptions.serialize(QueryType.REMOVE, localOptions, start, stop);
if (this.httpClient.apiVersion && this.httpClient.apiVersion["1"] >= 18) {
queryUrl = `/io/${this.name}/q`;
queryBody = QueryOptions.serialize(QueryType.REMOVE, localOptions, start, stop, entries);
} else if (entries.length > 1) throw new Error("Multiple entries require ReductStore API version >= 1.18");
const { data } = await this.httpClient.post(queryUrl, queryBody);
return Promise.resolve(data["removed_records"]);
}
/**
* Start writing a record into an entry
* @param entry name of the entry
* @param options {BigInt | WriteOptions} timestamp in microseconds for the record or options. It is current time if undefined.
* @return Promise<WritableRecord>
* @example
* const record = await bucket.beginWrite("entry", {
* ts: 12345667n
* labels: {label1: "value1", label2: "value2"}
* contentType: "text/plain"
* );
* await record.write("Hello!");
*/
async beginWrite(entry, options) {
let localOptions = { ts: void 0 };
if (options !== void 0) if (typeof options === "bigint") localOptions = { ts: options };
else localOptions = options;
localOptions.ts = localOptions.ts ?? BigInt(Date.now()) * 1000n;
return Promise.resolve(new WritableRecord(this.name, entry, localOptions, this.httpClient));
}
/**
* Update labels of an existing record
*
* If a label has empty string value, it will be removed.
*
* @param entry {string} name of the entry
* @param ts {BigInt} timestamp of record in microseconds
* @param labels {LabelMap} labels to update
*/
async update(entry, ts, labels) {
const headers = {};
for (const [key, value] of Object.entries(labels)) headers[`x-reduct-label-${key}`] = value.toString();
await this.httpClient.patch(`/b/${this.name}/${entry}?ts=${ts}`, "", headers);
}
/**
* Start reading a record from an entry
* @param entry name of the entry
* @param ts {BigInt} timestamp of record in microseconds. Get the latest one, if undefined
* @param head {boolean} return only head of the record
* @return Promise<ReadableRecord>
*/
async beginRead(entry, ts, head) {
return await this.readRecord(entry, head ?? false, ts ? ts.toString() : void 0);
}
/**
* Rename an entry
* @param entry entry name to rename
* @param newEntry new entry name
*/
async renameEntry(entry, newEntry) {
await this.httpClient.put(`/b/${this.name}/${entry}/rename`, { new_name: newEntry }, { "Content-Type": "application/json" });
}
/**
* Rename a bucket
* @param newName new name of the bucket
*/
async rename(newName) {
await this.httpClient.put(`/b/${this.name}/rename`, { new_name: newName }, { "Content-Type": "application/json" });
this.name = newName;
}
/**
* Query records for a time interval as generator
* @param entry {string | string[]} name of the entry or entries
* @param start {BigInt} start point of the time period
* @param stop {BigInt} stop point of the time period
* @param options {QueryOptions} options options for query
* @example
* for await (const record in bucket.query("entry-1", start, stop)) {
* console.log(record.ts, record.size);
* console.log(record.labels);
* const content = await record.read();
* // or use pipe
* const fileStream = fs.createWriteStream(`ts_${record.size}.txt`);
* record.pipe(fileStream);
* }
*/
async *query(entry, start, stop, options) {
const _options = options ?? {};
const continuous = _options.continuous ?? false;
const pollInterval = _options.pollInterval ?? 1;
const head = _options.head ?? false;
const entries = Array.isArray(entry) ? entry : [entry];
let fetcher = fetchAndParseBatchV1;
let queryUrl = `/b/${this.name}/${entry}/q`;
let queryBody = QueryOptions.serialize(QueryType.QUERY, _options, start, stop);
if (this.httpClient.apiVersion && this.httpClient.apiVersion["1"] >= 18) {
fetcher = fetchAndParseBatchV2;
queryUrl = `/io/${this.name}/q`;
queryBody = QueryOptions.serialize(QueryType.QUERY, _options, start, stop, entries);
} else if (entries.length > 1) throw new Error("Multiple entries require ReductStore API version >= 1.18");
const { data } = await this.httpClient.post(queryUrl, queryBody);
const { id } = data;
yield* fetcher(this.name, entries[0] ?? "", id, continuous, pollInterval, head, this.httpClient);
}
getName() {
return this.name;
}
async readRecord(entry, head, ts, id) {
const params = new URLSearchParams();
if (ts) params.set("ts", ts);
if (id) params.set("q", id);
const url = `/b/${this.name}/${entry}?${params.toString()}`;
const response = head ? await this.httpClient.head(url) : await this.httpClient.get(url);
if (response.status === 204) throw new APIError(response.headers.get("x-reduct-error") ?? "No content", 204);
const { headers, data } = response;
const labels = {};
for (const [key, value] of headers.entries()) if (key.startsWith("x-reduct-label-")) labels[key.slice(15)] = value;
const contentType = headers.get("content-type") ?? "application/octet-stream";
const contentLength = BigInt(headers.get("content-length") ?? 0);
const timestamp = BigInt(headers.get("x-reduct-time") ?? 0);
const last = true;
let stream;
if (head) stream = new ReadableStream({ start(ctrl) {
ctrl.close();
} });
else if (data instanceof ReadableStream) stream = data;
else if (data instanceof Blob) stream = data.stream();
else if (data instanceof Uint8Array) stream = new ReadableStream({ start(ctrl) {
ctrl.enqueue(data);
ctrl.close();
} });
else if (typeof data === "string") {
const bytes = new TextEncoder().encode(data);
stream = new ReadableStream({ start(ctrl) {
ctrl.enqueue(bytes);
ctrl.close();
} });
} else throw new Error("Invalid body type returned by httpClient");
return new ReadableRecord(entry, timestamp, contentLength, last, head, stream, labels, contentType);
}
/**
* Create a new batch for writing records to the database.
* @param entry
*/
async beginWriteBatch(entry) {
return new Batch(this.name, entry, this.httpClient, BatchType.WRITE);
}
/**
* Create a new batch for writing records to multiple entries.
* @example
* const batch = await bucket.beginWriteRecordBatch();
* batch.add("entry-1", 1000n, "data");
* batch.add("entry-2", 2000n, "data");
* await batch.send();
*/
beginWriteRecordBatch() {
return new RecordBatch(this.name, this.httpClient, RecordBatchType.WRITE);
}
/**
* Create a new batch for updating records in the database.
* @param entry
*/
async beginUpdateBatch(entry) {
return new Batch(this.name, entry, this.httpClient, BatchType.UPDATE);
}
/**
* Create a new batch for updating records across multiple entries.
* @example
* const batch = bucket.beginUpdateRecordBatch();
* batch.addOnlyLabels("entry-1", 1000n, { label1: "value1", label2: "" });
* batch.addOnlyLabels("entry-2", 2000n, { label1: "value2" });
* await batch.send();
*/
beginUpdateRecordBatch() {
return new RecordBatch(this.name, this.httpClient, RecordBatchType.UPDATE);
}
/**
* Create a new batch for removing records across multiple entries.
*/
beginRemoveRecordBatch() {
return new RecordBatch(this.name, this.httpClient, RecordBatchType.REMOVE);
}
/**
* Create a query link for downloading records
* @param entry name of the entry or entries
* @param start start point of the time period for the query
* @param stop stop point of the time period for the query
* @param query options for the query
* @param record selector for the record to download (required):
* - `number`: legacy record index (works only before ReductStore v1.19; removed from v1.19 API because broken, removed in SDK v1.21)
* - `{ entry, timestamp }`: explicit record identity for ReductStore v1.19+
* @param expireAt expiration time of the link. Default is 24 hours from now
* @param fileName name of the file to download. Default is `${entry}_<selector>.bin` or `${bucket}_<selector>.bin` for multi-entry
* @param baseUrl base url for link generation. If not set, the server's base url will be used
*/
async createQueryLink(entry, start, stop, query, record, expireAt, fileName, baseUrl) {
let selectedRecordIndex;
let selectedRecordEntry;
let selectedRecordTimestamp;
if (record === void 0) throw new Error("record selector must be provided (legacy index or { entry, timestamp })");
else if (typeof record === "number") {
if (this.httpClient.apiVersion && this.httpClient.apiVersion[1] >= 19) throw new Error("Numeric record index selector was removed from ReductStore v1.19 API because it is broken. Use { entry, timestamp }.");
if (!Number.isInteger(record) || record < 0) throw new Error("record index must be a non-negative integer");
selectedRecordIndex = record;
} else {
if (!record.entry) throw new Error("record entry must be provided");
if (record.timestamp === void 0) throw new Error("record timestamp must be provided");
selectedRecordEntry = record.entry;
selectedRecordTimestamp = record.timestamp;
}
const entries = Array.isArray(entry) ? entry : [entry];
const entryName = entries[0] ?? "";
if (Array.isArray(entry)) {
if (this.httpClient.apiVersion && this.httpClient.apiVersion["1"] < 18) throw new Error("Multiple entries require ReductStore API version >= 1.18");
if (entries.length == 0) throw new Error("At least one entry must be specified");
}
const queryLinkOptions = {
bucket: this.name,
entry: entryName,
recordEntry: selectedRecordEntry,
recordTimestamp: selectedRecordTimestamp,
query: query ?? {},
index: selectedRecordIndex,
expireAt: expireAt ?? new Date(Date.now() + 24 * 3600 * 1e3),
baseUrl
};
const entriesForQuery = Array.isArray(entry) ? entries : [];
const file = fileName ?? `${entryName.length == 0 ? this.name : entryName}_${selectedRecordIndex ?? selectedRecordTimestamp ?? 0}.bin`;
const { data } = await this.httpClient.post(`/links/${file}`, QueryLinkOptions.serialize(queryLinkOptions, start, stop, entriesForQuery));
return data.link;
}
/**
* Write attachments to an entry.
*
* Attachments are stored as JSON records in `${entry}/$meta` with `key` label.
*
* @param entry name of the source entry
* @param attachments map of attachment key to JSON-serializable content
*/
async writeAttachments(entry, attachments) {
const batch = this.beginWriteRecordBatch();
const entryName = `${entry}/$meta`;
const baseTs = BigInt(Date.now()) * 1000n;
let offset = 0n;
for (const [key, content] of Object.entries(attachments)) {
batch.add(entryName, baseTs + offset, JSON.stringify(content), "application/json", { key });
offset += 1n;
}
await batch.send();
}
/**
* Read attachments from an entry.
*
* @param entry name of the source entry
* @return map of attachment key to decoded JSON value
*/
async readAttachments(entry) {
const attachments = {};
const entryName = `${entry}/$meta`;
for await (const record of this.query(entryName)) {
if (record.labels["key"] === void 0) continue;
const key = record.labels["key"].toString();
attachments[key] = JSON.parse(await record.readAsString());
}
return attachments;
}
/**
* Remove attachments from an entry.
*
* If `attachmentKeys` is omitted, remove all attachments.
*
* @param entry name of the source entry
* @param attachmentKeys list of keys to remove
*/
async removeAttachments(entry, attachmentKeys) {
const entryName = `${entry}/$meta`;
const batch = this.beginUpdateRecordBatch();
const escapedAttachmentKeys = attachmentKeys?.map((key) => key.startsWith("$") ? `$${key}` : key);
const when = escapedAttachmentKeys && escapedAttachmentKeys.length > 0 ? { $in: [{ "&key": { $cast: "string" } }, ...escapedAttachmentKeys] } : {};
for await (const attachment of this.query(entryName, void 0, void 0, { when })) batch.addOnlyLabels(entryName, attachment.time, {
...attachment.labels,
remove: "true"
});
await batch.send();
}
};
//#endregion
export { Bucket };