@maximai/maxim-js
Version:
Maxim AI JS SDK. Visit https://getmaxim.ai for more info.
432 lines • 19.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LogWriter = void 0;
const uuid_1 = require("uuid");
const attachment_1 = require("../apis/attachment");
const logs_1 = require("../apis/logs");
const platform_1 = require("../platform");
const mutex_1 = require("../utils/mutex");
const queue_1 = require("../utils/queue");
const utils_1 = require("../utils/utils");
const attachment_2 = require("./components/attachment");
const types_1 = require("./components/types");
class LogWriter {
constructor(config) {
this.id = (0, utils_1.generateUniqueId)();
this.queue = new queue_1.Queue();
this.attachmentQueue = new queue_1.Queue();
this.storageQueue = new queue_1.Queue();
this.mutex = mutex_1.Mutex.get(`maxim-logs-${this.id}`);
this.flushInterval = null;
this.logsDir = `${platform_1.platform.tmpdir()}/maxim-sdk/${this.id}/maxim-logs`;
this.STORAGE_LOG_THRESHOLD = 900000; // ~900KB
this.config = config;
this.isDebug = config.isDebug || false;
this._raiseExceptions = config.raiseExceptions;
this.maxInMemoryLogs = config.maxInMemoryLogs || 100;
this.cache = config.cache;
this.logsAPIService = new logs_1.MaximLogsAPI(config.baseUrl, config.apiKey, config.isDebug);
this.attachmentAPIService = new attachment_1.MaximAttachmentAPI(config.baseUrl, config.apiKey, config.isDebug);
if (config.autoFlush) {
this.flushInterval = platform_1.platform.timers.setInterval(() => {
this.flush();
this.flushStorageLogs();
this.flushAttachments();
}, config.flushInterval ? config.flushInterval * 1000 : 10000);
// Call unref() to tell Node.js that this interval should not keep the process alive
platform_1.platform.timers.maybeUnref(this.flushInterval);
}
}
get writerConfig() {
return this.config;
}
get raiseExceptions() {
return this._raiseExceptions;
}
get writerLogsAPIService() {
return this.logsAPIService;
}
get writerCache() {
return this.cache;
}
isOnAWSLambda() {
return process.env["AWS_LAMBDA_FUNCTION_NAME"] !== undefined;
}
hasAccessToFilesystem() {
return platform_1.platform.fs.hasAccessToFilesystem();
}
writeToFile(logs) {
try {
return new Promise((resolve, reject) => {
if (!platform_1.platform.fs.existsSync(this.logsDir)) {
platform_1.platform.fs.mkdirpSync(this.logsDir);
}
const content = logs.map((l) => l.serialize()).join("\n");
const filename = `logs-${new Date().toISOString()}.log`;
platform_1.platform.fs
.writeFile(`${this.logsDir}/${filename}`, content)
.then(() => {
resolve(`${this.logsDir}/${filename}`);
})
.catch((err) => {
reject(err);
});
});
}
catch (err) {
if (this._raiseExceptions) {
throw err;
}
else {
console.error(`[Maxim-SDK] Error while writing to file: ${err instanceof Error ? err.message : err}`);
return undefined;
}
}
}
async flushLogFiles() {
if (!this.hasAccessToFilesystem() || !platform_1.platform.fs.existsSync(this.logsDir)) {
return;
}
const files = platform_1.platform.fs.readdirSync(this.logsDir);
await Promise.all(files.map(async (file) => {
const logs = platform_1.platform.fs.readFileSync(`${this.logsDir}/${file}`).toString();
// Now will push these to the server
try {
await this.logsAPIService.pushLogs(this.config.repositoryId, logs);
try {
platform_1.platform.fs.rmSync(`${this.logsDir}/${file}`);
}
catch (ignored) { }
}
catch (err) {
if (err && typeof err === "object" && "message" in err && typeof err.message === "string")
console.error(`Error while pushing logs: ${err.message}`);
}
}));
}
async uploadFile(attachmentData, entity, entityId) {
var _a;
try {
if (!attachmentData.path) {
console.error("[MaximSDK] Path is not set for file attachment. Skipping upload");
return;
}
// Detect mimetype if not provided
const mimeType = attachmentData.mimeType || platform_1.platform.mime.lookup(attachmentData.path) || "application/octet-stream";
const size = platform_1.platform.fs.statSync(attachmentData.path).size;
const key = attachmentData.key;
const data = platform_1.platform.fs.readFileSync(attachmentData.path);
// Get upload URL
const resp = await this.attachmentAPIService.getUploadUrl(key, mimeType, size);
// Create a copy of attachment without the path
const addAttachmentData = { ...attachmentData };
delete addAttachmentData.path;
// Create CommitLog for add-attachment action
const addAttachmentLog = new types_1.CommitLog(entity, entityId, "add-attachment", addAttachmentData);
// Queue it
this.queue.enqueue(addAttachmentLog);
// Uploading file to the Maxim API
await this.attachmentAPIService.uploadToSignedUrl(resp.url, Buffer.from(data), mimeType);
if (this.isDebug) {
console.log(`[MaximSDK] File uploaded to the Maxim API. URL: ${resp.url}, Mime type: ${mimeType}, Size: ${size}`);
}
}
catch (err) {
const currentRetry = "retry" in attachmentData && typeof attachmentData.retry === "number" ? ((_a = attachmentData.retry) !== null && _a !== void 0 ? _a : 0) : 0;
if (currentRetry < 3) {
attachmentData.retry = currentRetry + 1;
const retryLog = new types_1.CommitLog(entity, entityId, "upload-attachment", attachmentData);
this.attachmentQueue.enqueue(retryLog);
}
else {
console.error(`[MaximSDK] Failed to upload file: ${err instanceof Error ? err.message : err}`);
}
}
}
async uploadFileData(attachmentData, entity, entityId) {
var _a;
try {
if (!attachmentData.data) {
console.error("[MaximSDK] Data is not set for file data attachment. Skipping upload");
return;
}
const mimeType = attachmentData.mimeType || "application/octet-stream";
const key = attachmentData.key;
const size = attachmentData.data.length;
// Get upload URL
const resp = await this.attachmentAPIService.getUploadUrl(key, mimeType, size);
// Create a copy of attachment without the data
const addAttachmentData = { ...attachmentData };
delete addAttachmentData.data;
// Create CommitLog for add-attachment action
const addAttachmentLog = new types_1.CommitLog(entity, entityId, "add-attachment", addAttachmentData);
// Queue it
this.queue.enqueue(addAttachmentLog);
// Uploading file data to the Maxim API
await this.attachmentAPIService.uploadToSignedUrl(resp.url, attachmentData.data, mimeType);
if (this.isDebug) {
console.log(`[MaximSDK] File data uploaded to the Maxim API. URL: ${resp.url}, Mime type: ${mimeType}, Size: ${size}`);
}
}
catch (err) {
const currentRetry = "retry" in attachmentData && typeof attachmentData.retry === "number" ? ((_a = attachmentData.retry) !== null && _a !== void 0 ? _a : 0) : 0;
if (currentRetry < 3) {
attachmentData.retry = currentRetry + 1;
const retryLog = new types_1.CommitLog(entity, entityId, "upload-attachment", attachmentData);
this.attachmentQueue.enqueue(retryLog);
}
else {
console.error(`[MaximSDK] Failed to upload file data: ${err instanceof Error ? err.message : err}`);
}
}
}
async uploadAttachment(attachment) {
const entity = attachment.type;
const entityId = attachment.id;
const attachmentData = attachment.data;
const populatedAttachment = (0, attachment_2.populateAttachmentFields)(attachmentData);
const attachmentType = populatedAttachment.type;
switch (attachmentType) {
case "file":
await this.uploadFile(populatedAttachment, entity, entityId);
break;
case "fileData":
await this.uploadFileData(populatedAttachment, entity, entityId);
break;
case "url":
// For URL attachments, we just need to add them to the queue for sending to the server
const addAttachmentLog = new types_1.CommitLog(entity, entityId, "add-attachment", populatedAttachment);
this.queue.enqueue(addAttachmentLog);
break;
default:
const exhaustiveCheck = attachmentType;
console.error(`[MaximSDK] Unknown attachment type: ${attachmentType}. Skipping upload.`);
}
}
async flushAttachments() {
const attachments = this.attachmentQueue.dequeueAll();
if (attachments.length === 0) {
return;
}
await Promise.all(attachments.map(async (attachment) => {
return this.uploadAttachment(attachment);
}));
}
async flushLogs(logs) {
try {
// We can try to flush old failed logs first
await this.flushLogFiles();
if (this.isDebug) {
console.log("[MaximSDK] Flushing new logs");
logs.map((l) => console.log(l.serialize()));
}
// Flushing new logs
// Split logs into chunks of 5MB max
const MAX_SIZE = 5242880; // 5MB in bytes
const chunks = [];
let currentChunk = "";
for (const log of logs) {
const serialized = log.serialize() + "\n";
if (currentChunk.length + serialized.length > MAX_SIZE) {
chunks.push(currentChunk);
currentChunk = serialized;
}
else {
currentChunk += serialized;
}
}
if (currentChunk.length > 0) {
chunks.push(currentChunk);
}
// Make multiple requests if needed
for (const chunk of chunks) {
await this.logsAPIService.pushLogs(this.config.repositoryId, chunk);
if (this.isDebug)
console.log(`[MaximSDK] Flushed chunk of size ${chunk.length} bytes`);
}
// Return early since we've already made the API calls
return;
}
catch (err) {
console.error("Error while pushing logs", err);
if (this.isOnAWSLambda() || !this.hasAccessToFilesystem()) {
// Here we don't write it to filesystem
this.queue.enqueueAll(logs);
return;
}
await this.writeToFile(logs);
}
}
getAttachmentKey(log) {
if (log.action === "upload-attachment") {
const repoId = this.config.repositoryId;
const entity = log.type;
const entityId = log.id;
const attachmentData = log.data;
const fileId = attachmentData.id;
return `${repoId}/${entity}/${entityId}/files/original/${fileId}`;
}
return null;
}
async uploadStorageLog(log) {
var _a;
try {
if (!log.data || !log.data["logContent"]) {
console.error("[MaximSDK] Log content is not set for storage upload. Skipping upload.");
return;
}
const logContent = log.data["logContent"];
const storageId = (0, uuid_1.v4)();
const key = `${this.config.repositoryId}/large-logs/${storageId}`;
const resp = await this.attachmentAPIService.getUploadUrl(key, "text/plain", Buffer.byteLength(logContent, "utf8"));
await this.attachmentAPIService.uploadToSignedUrl(resp.url, Buffer.from(logContent, "utf8"), "text/plain");
const storageLog = new types_1.CommitLog(types_1.Entity.STORAGE, storageId, "process-large-log", { key });
this.queue.enqueue(storageLog);
if (this.isDebug) {
console.log(`[MaximSDK] Large log uploaded to storage. Key: ${key}, Size: ${logContent.length} bytes`);
}
}
catch (err) {
const currentRetry = "retry" in log.data && typeof log.data["retry"] === "number" ? ((_a = log.data["retry"]) !== null && _a !== void 0 ? _a : 0) : 0;
if (currentRetry < 3) {
log.data["retry"] = currentRetry + 1;
this.storageQueue.enqueue(log);
}
else {
console.error(`[MaximSDK] Failed to upload large log to storage: ${err instanceof Error ? err.message : err}`);
}
}
}
async flushStorageLogs() {
const storageLogs = this.storageQueue.dequeueAll();
if (storageLogs.length === 0) {
return;
}
await Promise.all(storageLogs.map(async (log) => {
return this.uploadStorageLog(log);
}));
}
commit(log) {
try {
const serializedLog = log.serialize();
if (this.isDebug)
console.log("[MaximSDK] Committing log: ", serializedLog);
if (!/^[a-zA-Z0-9_-]+$/.test(log.id)) {
if (this._raiseExceptions) {
throw new Error(`Invalid ID: ${log.id}. ID must only contain alphanumeric characters, hyphens, and underscores. Event will not be logged.`);
}
return;
}
// Check if this is a large log that should be uploaded to storage
if (Buffer.byteLength(serializedLog, "utf8") > this.STORAGE_LOG_THRESHOLD && log.action !== "upload-attachment") {
const storageLog = new types_1.CommitLog(log.type, log.id, "upload-storage-log", { logContent: serializedLog });
this.storageQueue.enqueue(storageLog);
}
// Special handling for upload-attachment action
else if (log.action === "upload-attachment") {
if (!log.data) {
console.error("[MaximSDK] Attachment data is not set for log. Skipping upload.");
return;
}
// Attach key
const key = this.getAttachmentKey(log);
if (key) {
// Add key to attachment data
log.data.key = key;
// Add to upload queue
this.attachmentQueue.enqueue(log);
}
else {
console.error(`[MaximSDK] Failed to generate attachment key due to invalid action: ${log.action}. Skipping upload.`);
}
}
else {
this.queue.enqueue(log);
}
if (this.queue.size + this.attachmentQueue.size + this.storageQueue.size > this.maxInMemoryLogs) {
this.flush();
}
}
catch (err) {
if (this._raiseExceptions) {
throw err;
}
else {
console.error(`[Maxim-SDK] Error while committing log: ${err instanceof Error ? err.message : err}`);
}
}
}
async flush() {
try {
let items = [];
// Add a timeout to the mutex lock to prevent deadlocks
const MUTEX_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes
// Create a promise that resolves after the timeout
let timeoutId;
const timeoutPromise = new Promise((resolve) => {
timeoutId = setTimeout(() => {
console.warn(`[MaximSDK] Mutex acquisition timed out after ${MUTEX_TIMEOUT_MS}ms. Skipping flush.`);
resolve();
}, MUTEX_TIMEOUT_MS);
});
// Race between the mutex lock and the timeout
await Promise.race([
new Promise(async (resolve) => {
await this.mutex.withLock(async () => {
try {
await this.flushStorageLogs();
await this.flushAttachments();
items = this.queue.dequeueAll();
if (items.length === 0) {
await this.flushLogFiles();
if (this.isDebug)
console.log("[MaximSDK] No logs to flush");
resolve();
}
if (this.isDebug)
console.log("[MaximSDK] Flushing logs");
await this.flushLogs(items);
}
catch (err) {
console.error("[MaximSDK] Couldn't flush logs", err);
resolve();
}
});
clearTimeout(timeoutId);
resolve();
}),
timeoutPromise,
]);
if (this.isDebug)
console.log("[MaximSDK] Flush complete");
}
catch (err) {
if (this._raiseExceptions) {
throw err;
}
else {
console.error(`[Maxim-SDK] Error while flushing logs: ${err instanceof Error ? err.message : err}`);
}
}
}
async cleanup() {
try {
if (this.flushInterval)
platform_1.platform.timers.clearInterval(this.flushInterval);
await this.flush();
// Destroy the HTTP/HTTPS agents to close any lingering connections
this.logsAPIService.destroyAgents();
this.attachmentAPIService.destroyAgents();
}
catch (err) {
if (this._raiseExceptions) {
throw err;
}
else {
console.error(`[Maxim-SDK] Error while cleaning up: ${err instanceof Error ? err.message : err}`);
}
}
}
}
exports.LogWriter = LogWriter;
//# sourceMappingURL=writer.js.map