@proddata/node-cratedb
Version:
Node.js client for CrateDB
433 lines • 19.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CrateDBClient = void 0;
const http_1 = __importDefault(require("http"));
const https_1 = __importDefault(require("https"));
const url_1 = require("url");
const Cursor_js_1 = require("./Cursor.js");
const Serializer_js_1 = require("./Serializer.js");
const Error_js_1 = require("./utils/Error.js");
const StatementGenerator_js_1 = require("./StatementGenerator.js");
const zlib_1 = __importDefault(require("zlib"));
const util_1 = require("util");
// Configuration options with CrateDB-specific environment variables
const defaultConfig = {
user: process.env.CRATEDB_USER || 'crate',
password: process.env.CRATEDB_PASSWORD || '',
jwt: null, // JWT token for Bearer authentication
host: process.env.CRATEDB_HOST || 'localhost',
port: process.env.CRATEDB_PORT ? parseInt(process.env.CRATEDB_PORT, 10) : 4200, // Default CrateDB port
connectionString: null,
ssl: false,
defaultSchema: process.env.CRATEDB_DEFAULT_SCHEMA || null, // Default schema for queries
keepAlive: true, // Enable persistent connections by default
maxConnections: 20,
deserialization: {
long: 'number',
timestamp: 'date',
date: 'date',
},
rowMode: 'array',
compression: {
request: 'gzip',
response: 'none',
},
};
class CrateDBClient {
cfg;
httpAgent;
protocol;
httpOptions;
constructor(config = {}) {
const cfg = { ...defaultConfig, ...config };
// Parse connection string if provided
if (cfg.connectionString) {
const parsed = new url_1.URL(cfg.connectionString);
cfg.user = cfg.user || parsed.username;
cfg.password = cfg.password || parsed.password;
cfg.host = parsed.hostname;
cfg.port = cfg.port || parseInt(parsed.port, 10);
cfg.ssl = cfg.ssl || parsed.protocol === 'https:';
}
// Set up HTTP(S) agent options based on configuration
const agentOptions = {
keepAlive: cfg.keepAlive,
maxSockets: cfg.maxConnections,
maxFreeSockets: cfg.maxConnections,
scheduling: 'fifo',
};
this.cfg = cfg;
this.httpAgent = cfg.ssl ? new https_1.default.Agent(agentOptions) : new http_1.default.Agent(agentOptions);
this.protocol = cfg.ssl ? 'https' : 'http';
// Determine authentication header: use JWT if available, else basic auth
let authHeader = {};
if (cfg.jwt) {
authHeader = { Authorization: `Bearer ${cfg.jwt}` };
}
else if (cfg.user && cfg.password) {
authHeader = {
Authorization: `Basic ${Buffer.from(`${cfg.user}:${cfg.password}`).toString('base64')}`,
};
}
const additionalHeaders = {};
if (cfg.compression.request == 'gzip') {
additionalHeaders['Content-Encoding'] = 'gzip';
}
if (cfg.compression.response == 'gzip') {
additionalHeaders['Accept-Encoding'] = 'gzip';
}
this.httpOptions = {
hostname: cfg.host,
port: cfg.port,
path: '/_sql?types',
method: 'POST',
headers: {
Connection: 'keep-alive',
'Content-Type': 'application/json',
Accept: 'application/json',
...authHeader,
...additionalHeaders,
...(cfg.defaultSchema ? { 'Default-Schema': cfg.defaultSchema } : {}),
},
auth: cfg.jwt ? undefined : cfg.user && cfg.password ? `${cfg.user}:${cfg.password}` : undefined,
agent: this.httpAgent,
};
}
createCursor(sql) {
return new Cursor_js_1.Cursor(this, sql);
}
async *streamQuery(sql, batchSize = 100) {
const cursor = this.createCursor(sql);
try {
await cursor.open();
yield* cursor.iterate(batchSize);
}
finally {
await cursor.close();
}
}
async execute(stmt, args, config) {
const startRequestTime = Date.now();
const payload = args ? { stmt, args } : { stmt };
let body;
try {
body = Serializer_js_1.Serializer.serialize(payload);
}
catch (serializationError) {
const msg = serializationError instanceof Error ? serializationError.message : String(serializationError);
throw new Error_js_1.RequestError(`Serialization failed: ${msg}`);
}
const options = {
...this.httpOptions,
...config?.httpOptions,
body,
};
try {
const response = await this._makeRequest(options);
const transformedResponse = this._transformResponse(response, config?.rowMode ?? this.cfg.rowMode);
return this._addDurations(startRequestTime, transformedResponse);
}
catch (error) {
if (error instanceof Error_js_1.CrateDBError || error instanceof Error_js_1.DeserializationError) {
throw error;
}
else if (error instanceof Error) {
throw new Error_js_1.RequestError(`CrateDB request failed: ${error.message}`, { cause: error });
}
throw new Error_js_1.RequestError('CrateDB request failed with an unknown error');
}
}
async executeMany(stmt, bulk_args) {
const startRequestTime = Date.now();
let body;
try {
body = Serializer_js_1.Serializer.serialize({ stmt, bulk_args });
}
catch (serializationError) {
const msg = serializationError instanceof Error ? serializationError.message : String(serializationError);
throw new Error_js_1.RequestError(`Serialization failed: ${msg}`);
}
const options = { ...this.httpOptions, body };
try {
const response = await this._makeRequest(options);
const res = this._addDurations(startRequestTime, response);
// Mark bulk errors for each result where rowcount is -2
res.bulk_errors = (res.results || [])
.map((result, i) => (result.rowcount === -2 ? i : null))
.filter((i) => i !== null);
return res;
}
catch (error) {
if (error instanceof Error_js_1.CrateDBError || error instanceof Error_js_1.DeserializationError) {
throw error;
}
else if (error instanceof Error) {
throw new Error_js_1.RequestError(`CrateDB bulk request failed: ${error.message}`, { cause: error });
}
throw new Error_js_1.RequestError('CrateDB bulk request failed with an unknown error');
}
}
/**
* Inserts a single row into a specified table with optional primary key conflict resolution.
*
* If primaryKeys are provided, conflicts will be handled by updating the non-primary key
* fields of the conflicting row. If no primaryKeys are provided, conflicting rows will be skipped.
*
* @param {string} tableName - The name of the table to insert the row into.
* @param {Record<string, unknown>} obj - An object representing the row to insert.
* @param {string[] | null} [primaryKeys=null] - Array of column names to use as primary keys for conflict resolution.
* @returns {Promise<CrateDBResponse>} A promise resolving to the response from CrateDB.
* @throws {Error} If the input parameters are invalid.
*/
async insert(tableName, obj, primaryKeys = null) {
// Validate inputs
if (!tableName || typeof tableName !== 'string') {
throw new Error('tableName must be a valid string');
}
if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
throw new Error('obj must be a valid non-array object');
}
if (primaryKeys && !Array.isArray(primaryKeys)) {
throw new Error('primaryKeys must be an array or null');
}
const keys = Object.keys(obj);
const query = StatementGenerator_js_1.StatementGenerator.insert(tableName, keys, primaryKeys);
const args = Object.values(obj);
// Execute the query
return this.execute(query, args);
}
/**
* Inserts multiple rows into a table with optional primary key conflict resolution.
*
* If primaryKeys are provided, conflicts will be handled by updating the non-primary key
* fields of conflicting rows. If no primaryKeys are provided, conflicting rows will be skipped.
* The method automatically handles varying column sets across the input objects.
*
* @param {string} tableName - The name of the table to insert rows into.
* @param {Record<string, unknown>[]} objectArray - Array of objects representing rows to insert.
* @param {string[] | null} [primaryKeys=null] - Array of column names to use as primary keys for conflict resolution.
* @returns {Promise<CrateDBBulkResponse>} A promise resolving to the bulk operation response from CrateDB.
* @throws {Error} If the input parameters are invalid or if the operation fails.
*/
async insertMany(tableName, objectArray, primaryKeys = null) {
const startInsertMany = Date.now();
// Validate inputs
if (!tableName || typeof tableName !== 'string') {
throw new Error('tableName must be a valid string.');
}
if (!Array.isArray(objectArray) || objectArray.length === 0) {
throw new Error('insertMany requires a non-empty array of objects.');
}
if (primaryKeys && !Array.isArray(primaryKeys)) {
throw new Error('primaryKeys must be an array or null.');
}
// Extract unique keys from all objects
const uniqueKeys = Array.from(objectArray.reduce((keys, obj) => {
Object.keys(obj).forEach((key) => keys.add(key));
return keys;
}, new Set()));
// Generate bulk arguments
const bulkArgs = objectArray.map((obj) => uniqueKeys.map((key) => (Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : null)));
const query = StatementGenerator_js_1.StatementGenerator.insert(tableName, uniqueKeys, primaryKeys);
// Execute the query with bulk arguments
const response = await this.executeMany(query, bulkArgs);
const elapsedTime = Date.now() - startInsertMany;
response.durations.preparation = elapsedTime - response.durations.request - (response.durations.cratedb ?? 0);
return response;
}
/**
* Drops a table if it exists in CrateDB.
*
* Constructs and executes a `DROP TABLE IF EXISTS` SQL statement.
*
* @param {string} tableName - The name of the table to drop.
* @returns {Promise<CrateDBResponse>} A promise resolving to the response from CrateDB.
*/
async drop(tableName) {
const query = StatementGenerator_js_1.StatementGenerator.dropTable(tableName);
return this.execute(query);
}
async createTable(tableName, schema, options) {
const query = StatementGenerator_js_1.StatementGenerator.createTable(tableName, schema, options);
return this.execute(query);
}
/**
* Refreshes a given table by refreshing it in CrateDB.
*
* The `REFRESH TABLE` command makes recently committed changes available for querying
* without waiting for automatic refresh intervals.
*
* @param {string} tableName - The name of the table to refresh.
* @returns {Promise<CrateDBResponse>} A promise resolving to the response from CrateDB.
*/
async refresh(tableName) {
const query = StatementGenerator_js_1.StatementGenerator.refresh(tableName);
return this.execute(query);
}
/**
* Optimizes a given table or specific partitions in CrateDB by merging table segments.
*
* The `OPTIMIZE TABLE` command reduces the number of segments in a table, improving
* query performance and reducing storage overhead. It supports optimizing the entire table
* or specific partitions and allows additional optimization parameters.
*
* @param {string} tableName - The name of the table to optimize.
* @param {OptimizeOptions} [options] - Optional parameters for table optimization.
* @param {Record<string, string | number>} [partitions] - Optional key-value pairs specifying partition columns and values.
* @returns {Promise<CrateDBResponse>} A promise resolving to the response from CrateDB.
*/
async optimize(tableName, options, partitions) {
const query = StatementGenerator_js_1.StatementGenerator.optimize(tableName, options, partitions);
return this.execute(query);
}
/**
* Retrieves the primary key columns for a given table.
*
* Queries the information_schema to get the primary key columns
* of the specified table in their defined order.
*
* @param {string} tableName - The name of the table to get primary keys for.
* @returns {Promise<string[]>} A promise resolving to an array of primary key column names.
* @throws {Error} If the table doesn't exist or if there's an error retrieving the information.
*/
async getPrimaryKeys(tableName) {
if (!tableName || typeof tableName !== 'string') {
throw new Error('tableName must be a valid string');
}
// Split schema and table name
const [schema = 'doc', table] = tableName.split('.');
const actualTable = table || schema;
const actualSchema = table ? schema : 'doc';
const query = StatementGenerator_js_1.StatementGenerator.getPrimaryKeys();
const response = await this.execute(query, [actualSchema, actualTable]);
if (!response.rows || response.rows.length === 0) {
return [];
}
return response.rows.map((row) => row[0]);
}
_addDurations(startRequestTime, response) {
const totalTime = Date.now() - startRequestTime;
const cratedb = typeof response.duration === 'number' ? response.duration : 0;
response.durations = {
...response.durations, // preserves encoding, request, deserialization from _makeRequest
total: totalTime,
cratedb,
};
return response;
}
async _makeRequest(options) {
let requestBody = options.body;
const headers = { ...options.headers };
const requestBodySize = requestBody ? Buffer.byteLength(requestBody) : 0;
let encodedSize = requestBodySize;
let encodingDuration = 0;
let requestDuration = 0;
let deserializationDuration = 0;
// Compress the request body if needed.
if (this.cfg.compression.request === 'gzip' && requestBody) {
const startEncodingTime = Date.now();
requestBody = await (0, util_1.promisify)(zlib_1.default.gzip)(requestBody);
encodingDuration = Date.now() - startEncodingTime;
encodedSize = Buffer.byteLength(requestBody);
}
// Wrap the HTTP request in a promise.
const { response, data } = await new Promise((resolve, reject) => {
const protocolHandler = this.protocol === 'https' ? https_1.default : http_1.default;
const req = protocolHandler.request({ ...options, headers }, (response) => {
const data = [];
response.on('data', (chunk) => data.push(chunk));
response.on('end', () => resolve({ response, data }));
});
req.on('error', (err) => reject(new Error(`Request failed: ${err.message}\nStack trace: ${err.stack}`)));
req.end(requestBody);
});
requestDuration = Date.now() - (response.headers.date ? new Date(response.headers.date).getTime() : Date.now());
let rawResponse = Buffer.concat(data);
const responseBodySize = rawResponse.length;
// Decompress the response if gzip encoded.
if (response.headers['content-encoding'] === 'gzip') {
try {
rawResponse = await (0, util_1.promisify)(zlib_1.default.gunzip)(rawResponse);
}
catch (error) {
if (error instanceof Error) {
throw new Error(`Failed to decompress response: ${error.message}`);
}
else {
throw new Error(`Failed to decompress response: ${String(error)}`);
}
}
}
// Deserialize the response.
const startDeserializationTime = Date.now();
let parsedResponse;
try {
parsedResponse = Serializer_js_1.Serializer.deserialize(rawResponse.toString(), this.cfg.deserialization);
}
catch (err) {
if (err instanceof Error) {
throw new Error(`Failed to parse response: ${err.message}. Raw response: ${rawResponse.toString()}`);
}
else {
throw new Error(`Failed to parse response: ${String(err)}. Raw response: ${rawResponse.toString()}`);
}
}
deserializationDuration = Date.now() - startDeserializationTime;
if (response.statusCode === 200) {
return {
...parsedResponse,
sizes: {
response: responseBodySize,
request: encodedSize,
requestUncompressed: requestBodySize,
},
durations: {
encoding: encodingDuration,
request: requestDuration,
deserialization: deserializationDuration,
},
};
}
else {
throw Error_js_1.CrateDBError.fromResponse(parsedResponse, response.statusCode);
}
}
_transformResponse(response, rowMode = 'object') {
// Return early if not transforming to object mode
if (rowMode !== 'object') {
return response;
}
// Create a shallow copy of the response
const transformedResponse = { ...response };
// Only transform if we have both rows and column names
if (Array.isArray(transformedResponse.rows) && Array.isArray(transformedResponse.cols)) {
transformedResponse.rows = transformedResponse.rows.map((row) => {
// Skip transformation if row is null or not an array
if (!Array.isArray(row)) {
return row;
}
const obj = {};
transformedResponse.cols?.forEach((col, index) => {
// Only set property if column name is a string
if (typeof col === 'string') {
// Preserve null/undefined values
obj[col] = row[index];
}
});
return obj;
});
}
return transformedResponse;
}
getConfig() {
return this.cfg;
}
getHttpOptions() {
return this.httpOptions;
}
}
exports.CrateDBClient = CrateDBClient;
//# sourceMappingURL=CrateDBClient.js.map