winston-azure-blob
Version:
NEW winston transport for azure blob storage
300 lines • 14.3 kB
JavaScript
;
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.winstonAzureBlob = exports.WinstonAzureBlob = exports.extensions = void 0;
var winston_transport_1 = __importDefault(require("winston-transport"));
var async = __importStar(require("async"));
var moment_1 = __importDefault(require("moment"));
var triple_beam_1 = require("triple-beam");
var debug_1 = __importDefault(require("debug"));
var storage_blob_1 = require("@azure/storage-blob");
var debug = (0, debug_1.default)("winston-azure-blob");
var MAX_APPEND_BLOB_BLOCK_SIZE = 4 * 1024 * 1024;
/**
* File extensions for the log file. More can be added
*/
var extensions;
(function (extensions) {
extensions["LOG"] = ".log";
})(extensions = exports.extensions || (exports.extensions = {}));
/**
* Default options for constructing logger
*/
var loggerDefaults = {
// due to limitation of 50K block in azure blob storage we add some params to avoid the limit
bufferLogSize: -1,
eol: "\n",
extension: undefined,
rotatePeriod: "",
syncTimeout: 0, // The maximum time between two sync calls. Set to zero for realtime logging
};
//
// Extend from `winston-transport` to take advantage
// of the base functionality and `.exceptions.handle()`.
//
var WinstonAzureBlob = /** @class */ (function (_super) {
__extends(WinstonAzureBlob, _super);
function WinstonAzureBlob(opts) {
var _this = _super.call(this, opts) || this;
var options = __assign(__assign({}, loggerDefaults), opts);
WinstonAzureBlob.isValidAccountOpts(options.account);
_this.azBlobClient = WinstonAzureBlob.createAzBlobClient(options.account);
_this.blobName = options.blobName;
_this.buffer = [];
_this.bufferLogSize = options.bufferLogSize;
_this.containerName = options.containerName;
_this.eol = options.eol;
_this.extension = options.extension;
_this.headers = options.headers;
_this.rotatePeriod = options.rotatePeriod;
_this.syncTimeout = options.syncTimeout;
_this.timeoutFn = null;
if (_this.bufferLogSize > 1 && !_this.syncTimeout) {
throw new Error("syncTimeout must be set, if there is a bufferLogSize");
}
return _this;
}
WinstonAzureBlob.tackOnRotatePeriodToBlobName = function (_a) {
var blobName = _a.blobName, rotatePeriod = _a.rotatePeriod;
if (rotatePeriod) {
return blobName + "." + (0, moment_1.default)().format(rotatePeriod);
}
return blobName;
};
WinstonAzureBlob.tackOnExtensionToBlobName = function (_a) {
var blobName = _a.blobName, extension = _a.extension;
if (extension) {
return blobName + extension;
}
return blobName;
};
WinstonAzureBlob.generateBlobName = function (_a) {
var blobName = _a.blobName, extension = _a.extension, rotatePeriod = _a.rotatePeriod;
return WinstonAzureBlob.tackOnExtensionToBlobName({
blobName: WinstonAzureBlob.tackOnRotatePeriodToBlobName({
blobName: blobName,
rotatePeriod: rotatePeriod,
}),
extension: extension,
});
};
WinstonAzureBlob.createAzBlobClient = function (account_info) {
if ("key" in account_info) {
var sharedKeyCredential = new storage_blob_1.StorageSharedKeyCredential(account_info.name, account_info.key);
return new storage_blob_1.BlobServiceClient("https://".concat(account_info.name, ".blob.core.windows.net"), sharedKeyCredential);
}
if ("connectionString" in account_info) {
return storage_blob_1.BlobServiceClient.fromConnectionString(account_info.connectionString);
}
return new storage_blob_1.BlobServiceClient("".concat(account_info.host).concat(account_info.sasToken));
};
WinstonAzureBlob.chunkString = function (str, len) {
var size = Math.ceil(str.length / len);
var r = Array(size);
var offset = 0;
for (var i = 0; i < size; i++) {
r[i] = str.substring(offset, len);
offset += len;
}
return r;
};
WinstonAzureBlob.isValidAccountOpts = function (account) {
if ("key" in account &&
(typeof account.key !== "string" ||
typeof account.name !== "string")) {
throw new Error("Azure account key/name must be string values, received key:".concat(typeof account.key, ", name:").concat(typeof account.name, " "));
}
if ("connectionString" in account &&
typeof account.connectionString !== "string") {
throw new Error("Azure account connectionString must be a string value, received connectionString:".concat(typeof account.connectionString, " "));
}
if (!("key" in account || "connectionString" in account) &&
(typeof account.host !== "string" ||
typeof account.sasToken !== "string")) {
throw new Error("Azure account host/sasToken must be string values, received key:".concat(typeof account.host, ", name:").concat(typeof account.sasToken, " "));
}
};
WinstonAzureBlob.appendBlobOperation = function (appendBlobClient, chunk, nextAppendBlock, headers) {
return __awaiter(this, void 0, void 0, function () {
var succeeded, result, err_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 5, 6, 7]);
return [4 /*yield*/, appendBlobClient.createIfNotExists()];
case 1:
succeeded = (_a.sent()).succeeded;
if (!(succeeded && headers)) return [3 /*break*/, 3];
return [4 /*yield*/, appendBlobClient.appendBlock(headers.join(",") + "\n", headers.join(",").length)];
case 2:
_a.sent();
_a.label = 3;
case 3: return [4 /*yield*/, appendBlobClient.appendBlock(chunk, chunk.length)];
case 4:
result = _a.sent();
if (result.errorCode) {
debug(result.errorCode, result);
}
return [3 /*break*/, 7];
case 5:
err_1 = _a.sent();
debug(err_1);
return [3 /*break*/, 7];
case 6:
nextAppendBlock();
return [7 /*endfinally*/];
case 7: return [2 /*return*/];
}
});
});
};
WinstonAzureBlob.prototype.log = function (info, next) {
var _this = this;
this.push(info, function () {
_this.emit("logged", info);
next();
});
};
WinstonAzureBlob.prototype.push = function (data, next) {
var _this = this;
if (data) {
this.buffer.push(data);
}
if (this.bufferLogSize < 1 ||
this.buffer.length >= this.bufferLogSize) {
this.logToAppendBlob(this.buffer, next); // in this case winston buffer for us
this.buffer = [];
}
else if (this.syncTimeout && this.timeoutFn === null) {
this.timeoutFn = setTimeout(function () {
var tasks = _this.buffer.slice(0);
_this.buffer = [];
_this.timeoutFn = null; // as we can receive push again after timeout we must relaunch the timeout
_this.logToAppendBlob(tasks, function () {
debug("Finish to appendblock", tasks.length);
});
}, this.syncTimeout);
next();
}
else {
// buffering
next();
}
};
WinstonAzureBlob.prototype.logToAppendBlob = function (tasks, next) {
var _this = this;
debug("Try to appendblock", tasks.length);
// nothing to log
if (tasks.length === 0) {
return next();
}
var azClient = this.azBlobClient;
var containerName = this.containerName;
var containerClient = azClient.getContainerClient(containerName);
var blobName = WinstonAzureBlob.generateBlobName({
blobName: this.blobName,
extension: this.extension,
rotatePeriod: this.rotatePeriod,
});
var appendBlobClient = containerClient.getAppendBlobClient(blobName);
var toSend = tasks
// Symbol cannot be used as an index type. Therefore, a cast is required.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map(function (item) { return item[triple_beam_1.MESSAGE]; })
.join(this.eol) + this.eol;
var chunks = WinstonAzureBlob.chunkString(toSend, MAX_APPEND_BLOB_BLOCK_SIZE);
debug("Numbers of appendblock needed", chunks.length);
debug("Size of chunks", toSend.length);
async.eachSeries(chunks, function (chunk, nextAppendBlock) {
WinstonAzureBlob.appendBlobOperation(appendBlobClient, chunk, nextAppendBlock, _this.headers);
}, next);
};
return WinstonAzureBlob;
}(winston_transport_1.default));
exports.WinstonAzureBlob = WinstonAzureBlob;
var winstonAzureBlob = function (opts) { return new WinstonAzureBlob(opts); };
exports.winstonAzureBlob = winstonAzureBlob;
//# sourceMappingURL=winston-azure-blob.js.map