@gensx/storage
Version:
Cloud storage, blobs, sqlite, and vector database providers/hooks for GenSX.
509 lines (505 loc) • 19.9 kB
JavaScript
/**
* Check out the docs at https://www.gensx.com/docs
* Find us on Github https://github.com/gensx-inc/gensx
* Find us on Discord https://discord.gg/F5BSU8Kc
*/
import { Readable } from 'stream';
import { readConfig } from '@gensx/core';
import { parseErrorResponse } from '../utils/parse-error.js';
import { USER_AGENT } from '../utils/user-agent.js';
import { BlobInternalError, BlobError, BlobNetworkError, BlobConflictError } from './types.js';
/* eslint-disable @typescript-eslint/only-throw-error */
/**
* Base URL for the GenSX Console API
*/
const API_BASE_URL = "https://api.gensx.com";
/**
* Helper to convert between API errors and BlobErrors
*/
function handleApiError(err, operation) {
if (err instanceof BlobError) {
throw err;
}
if (err instanceof Error) {
throw new BlobNetworkError(`Error during ${operation}: ${err.message}`, err);
}
throw new BlobNetworkError(`Error during ${operation}: ${String(err)}`);
}
/**
* Implementation of Blob interface for remote cloud storage
*/
class RemoteBlob {
key;
baseUrl;
apiKey;
org;
project;
environment;
constructor(key, baseUrl, apiKey, org, project, environment) {
this.key = encodeURIComponent(key);
this.baseUrl = baseUrl;
this.apiKey = apiKey;
this.org = org;
this.project = project;
this.environment = environment;
}
async getJSON() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (response.status === 404) {
return null;
}
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to get blob: ${message}`);
}
const data = (await response.json());
// Parse the content as JSON since it's stored as a string
return JSON.parse(data.content);
}
catch (err) {
throw handleApiError(err, "getJSON");
}
}
async getString() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (response.status === 404) {
return null;
}
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to get blob: ${message}`);
}
const data = (await response.json());
return data.content;
}
catch (err) {
throw handleApiError(err, "getString");
}
}
async getRaw() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (response.status === 404) {
return null;
}
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to get blob: ${message}`);
}
const data = (await response.json());
const { content, contentType, etag, lastModified, size, metadata = {}, } = data;
// Always decode base64 for raw data
const buffer = Buffer.from(content, "base64");
return {
content: buffer,
contentType,
etag,
lastModified: lastModified ? new Date(lastModified) : undefined,
size,
metadata: Object.fromEntries(Object.entries(metadata).filter(([key]) => key !== "isBase64")),
};
}
catch (err) {
throw handleApiError(err, "getRaw");
}
}
async getStream() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to get blob: ${message}`);
}
if (!response.body) {
throw new BlobInternalError("Response body is null");
}
return Readable.from(response.body);
}
catch (err) {
throw handleApiError(err, "getStream");
}
}
async putJSON(value, options) {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "PUT",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
...(options?.etag && { "If-Match": options.etag }),
"User-Agent": USER_AGENT,
},
body: JSON.stringify({
content: JSON.stringify(value),
contentType: "application/json",
metadata: options?.metadata,
}),
});
if (!response.ok) {
if (response.status === 412) {
throw new BlobConflictError("ETag mismatch");
}
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to put blob: ${message}`);
}
const etag = response.headers.get("etag");
if (!etag) {
throw new BlobInternalError("No ETag returned from server");
}
return { etag };
}
catch (err) {
throw handleApiError(err, "putJSON");
}
}
async putString(value, options) {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "PUT",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
...(options?.etag && { "If-Match": options.etag }),
"User-Agent": USER_AGENT,
},
body: JSON.stringify({
content: value,
contentType: "text/plain",
metadata: options?.metadata,
}),
});
if (!response.ok) {
if (response.status === 412) {
throw new BlobConflictError("ETag mismatch");
}
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to put blob: ${message}`);
}
const etag = response.headers.get("etag");
if (!etag) {
throw new BlobInternalError("No ETag returned from server");
}
return { etag };
}
catch (err) {
throw handleApiError(err, "putString");
}
}
/**
* Put raw binary data into the blob.
* @param value The binary data to store
* @param options Optional metadata and content type
*/
async putRaw(value, options) {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "PUT",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
...(options?.etag && { "If-Match": options.etag }),
"User-Agent": USER_AGENT,
},
body: JSON.stringify({
content: value.toString("base64"),
contentType: options?.contentType ?? "application/octet-stream",
metadata: {
...(options?.metadata ?? {}),
isBase64: "true",
},
}),
});
if (!response.ok) {
if (response.status === 412) {
throw new BlobConflictError("ETag mismatch");
}
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to put blob: ${message}`);
}
const etag = response.headers.get("etag");
if (!etag) {
throw new BlobInternalError("No ETag returned from server");
}
return { etag };
}
catch (err) {
throw handleApiError(err, "putRaw");
}
}
async putStream(stream, options) {
try {
// Convert stream to buffer - necessary for the current API implementation
const chunks = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
const buffer = Buffer.concat(chunks);
// Send the buffer as base64-encoded content
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "PUT",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
...(options?.etag && { "If-Match": options.etag }),
"User-Agent": USER_AGENT,
},
body: JSON.stringify({
content: buffer.toString("base64"),
contentType: options?.contentType ?? "application/octet-stream",
metadata: {
...(options?.metadata ?? {}),
isBase64: "true",
},
}),
});
if (!response.ok) {
if (response.status === 412) {
throw new BlobConflictError("ETag mismatch");
}
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to put blob: ${message}`);
}
const etag = response.headers.get("etag");
if (!etag) {
throw new BlobInternalError("No ETag returned from server");
}
return { etag };
}
catch (err) {
throw handleApiError(err, "putStream");
}
}
async delete() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "DELETE",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (!response.ok && response.status !== 404) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to delete blob: ${message}`);
}
}
catch (err) {
throw handleApiError(err, "delete");
}
}
async exists() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "HEAD",
headers: {
Authorization: `Bearer ${this.apiKey}`,
},
});
return response.ok;
}
catch (err) {
throw handleApiError(err, "exists");
}
}
async getMetadata() {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "HEAD",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (response.status === 404) {
return null;
}
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to get metadata: ${message}`);
}
// Extract standard headers
const metadata = {};
// Get etag from standard headers
const etag = response.headers.get("etag");
if (etag) {
metadata.etag = etag;
}
// Get custom metadata from individual x-blob-meta-* headers
for (const [name, value] of Object.entries(Object.fromEntries(response.headers))) {
if (name.toLowerCase().startsWith("x-blob-meta-")) {
const metaKey = name.substring("x-blob-meta-".length);
if (metaKey === "content-type") {
metadata.contentType = value;
}
else {
metadata[metaKey] = value;
}
}
}
return Object.keys(metadata).length > 0 ? metadata : null;
}
catch (err) {
throw handleApiError(err, "getMetadata");
}
}
async updateMetadata(metadata, options) {
try {
const response = await fetch(`${this.baseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob/${this.key}`, {
method: "PATCH",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
...(options?.etag && { "If-Match": options.etag }),
"User-Agent": USER_AGENT,
},
body: JSON.stringify({
metadata,
}),
});
if (!response.ok) {
if (response.status === 412) {
throw new BlobConflictError("ETag mismatch");
}
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to update metadata: ${message}`);
}
}
catch (err) {
throw handleApiError(err, "updateMetadata");
}
}
}
/**
* Remote implementation of blob storage using GenSX Console API
*/
class RemoteBlobStorage {
apiKey;
apiBaseUrl;
org;
project;
environment;
defaultPrefix;
constructor(project, environment, defaultPrefix) {
this.project = project;
this.environment = environment;
this.defaultPrefix = defaultPrefix;
// readConfig has internal error handling and always returns a GensxConfig object
const config = readConfig();
this.apiKey = process.env.GENSX_API_KEY ?? config.api?.token ?? "";
if (!this.apiKey) {
throw new Error("GENSX_API_KEY environment variable must be set for cloud storage");
}
this.org = process.env.GENSX_ORG ?? config.api?.org ?? "";
if (!this.org) {
throw new Error("Organization must be provided via props or GENSX_ORG environment variable");
}
this.apiBaseUrl =
process.env.GENSX_API_BASE_URL ?? config.api?.baseUrl ?? API_BASE_URL;
}
getBlob(key) {
const fullKey = this.defaultPrefix ? `${this.defaultPrefix}/${key}` : key;
return new RemoteBlob(fullKey, this.apiBaseUrl, this.apiKey, this.org, this.project, this.environment);
}
async listBlobs(options) {
try {
// Normalize prefixes by removing trailing slashes
const normalizedDefaultPrefix = this.defaultPrefix?.replace(/\/$/, "");
const normalizedPrefix = options?.prefix?.replace(/\/$/, "");
// Build the search prefix
const searchPrefix = normalizedDefaultPrefix
? normalizedPrefix
? `${normalizedDefaultPrefix}/${normalizedPrefix}`
: normalizedDefaultPrefix
: (normalizedPrefix ?? "");
// Build query parameters
const queryParams = new URLSearchParams();
if (searchPrefix) {
queryParams.append("prefix", searchPrefix);
}
// Default to 100 documents if no limit is specified
queryParams.append("limit", (options?.limit ?? 100).toString());
if (options?.cursor) {
queryParams.append("cursor", options.cursor);
}
const response = await fetch(`${this.apiBaseUrl}/org/${this.org}/projects/${this.project}/environments/${this.environment}/blob?${queryParams.toString()}`, {
method: "GET",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"User-Agent": USER_AGENT,
},
});
if (!response.ok) {
const message = await parseErrorResponse(response);
throw new BlobInternalError(`Failed to list blobs: ${message}`);
}
const data = (await response.json());
// Process blob keys to handle the default prefix
let processedBlobs = [];
if (normalizedDefaultPrefix) {
processedBlobs = data.blobs
.filter((blob) => blob.key === normalizedDefaultPrefix ||
blob.key.startsWith(`${normalizedDefaultPrefix}/`))
.map((blob) => ({
...blob,
key: blob.key === normalizedDefaultPrefix
? ""
: blob.key.slice(normalizedDefaultPrefix.length + 1),
}));
}
else {
processedBlobs = data.blobs;
}
return {
blobs: processedBlobs,
nextCursor: data.nextCursor,
};
}
catch (err) {
if (err instanceof BlobError) {
throw err;
}
throw new BlobNetworkError(`Error during listBlobs operation: ${String(err)}`, err);
}
}
async blobExists(key) {
const blob = this.getBlob(key);
return blob.exists();
}
async deleteBlob(key) {
try {
const blob = this.getBlob(key);
await blob.delete();
return { deleted: true };
}
catch (err) {
if (err instanceof BlobError) {
throw err;
}
throw new BlobNetworkError(`Error during deleteBlob operation: ${String(err)}`, err);
}
}
}
export { RemoteBlob, RemoteBlobStorage };
//# sourceMappingURL=remote.js.map