@unito/integration-sdk
Version:
Integration SDK
1,250 lines (1,226 loc) • 115 kB
JavaScript
'use strict';
var integrationApi = require('@unito/integration-api');
var cachette = require('cachette');
var crypto = require('crypto');
var util = require('util');
var tracer = require('dd-trace');
var express = require('express');
var busboy = require('busboy');
var fs = require('fs');
var https = require('https');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var integrationApi__namespace = /*#__PURE__*/_interopNamespaceDefault(integrationApi);
const LOG_LEVELS = ['debug', 'info', 'warn', 'error', 'disabled'];
const LOG_LEVEL_SEVERITY = Object.fromEntries(LOG_LEVELS.map((name, i) => [name, i]));
const DEFAULT_LOG_LEVEL = 'info';
function readEnvLogLevel() {
const value = process.env.INTEGRATION_LOG_LEVEL;
if (value && LOG_LEVELS.includes(value)) {
return value;
}
return DEFAULT_LOG_LEVEL;
}
/**
* See https://docs.datadoghq.com/logs/log_collection/?tab=host#custom-log-forwarding
* - Datadog Agent splits at 256kB (256000 bytes)...
* - ... but the same docs say that "for optimal performance, it is
* recommended that an individual log be no greater than 25kB"
* -> Truncating at 25kB - a bit of wiggle room for metadata = 20kB.
*/
const MAX_LOG_MESSAGE_SIZE = parseInt(process.env.MAX_LOG_MESSAGE_SIZE ?? '20000', 10);
const LOG_LINE_TRUNCATED_SUFFIX = ' - LOG LINE TRUNCATED';
/**
* For *LogMeta* sanitization, we let in anything that was passed, except for clearly-problematic keys
*/
const LOGMETA_BLACKLIST = [
// Security
'access_token',
'bot_auth_code',
'client_secret',
'jwt',
'oauth_token',
'password',
'refresh_token',
'shared_secret',
'token',
// Privacy
'billing_email',
'email',
'first_name',
'last_name',
];
/**
* Logger class that can be configured with metadata add creation and when logging to add additional context to your logs.
*/
class Logger {
metadata;
level;
constructor(metadata = {}) {
this.metadata = structuredClone(metadata);
this.level = readEnvLogLevel();
}
/**
* Sets the minimum log level. Messages below this level are suppressed.
* @param level The minimum log level name.
*/
setLevel(level) {
this.level = level;
}
/**
* Logs an error message with the 'error' log level.
* @param message The error message to be logged.
* @param metadata Optional metadata to be associated with the log message.
*/
error(message, metadata) {
this.log('error', message, metadata);
}
/**
* Logs a warning message with the 'warn' log level.
* @param message The warning message to be logged.
* @param metadata Optional metadata to be associated with the log message.
*/
warn(message, metadata) {
this.log('warn', message, metadata);
}
/**
* Logs an informational message with the 'info' log level.
* @param message The informational message to be logged.
* @param metadata Optional metadata to be associated with the log message.
*/
info(message, metadata) {
this.log('info', message, metadata);
}
/**
* Logs a debug message with the 'debug' log level.
* @param message The debug message to be logged.
* @param metadata Optional metadata to be associated with the log message.
*/
debug(message, metadata) {
this.log('debug', message, metadata);
}
/**
* Decorates the logger with additional metadata.
* @param metadata Additional metadata to be added to the logger.
*/
decorate(metadata) {
this.metadata = { ...this.metadata, ...metadata };
}
/**
* Return a copy of the Logger's metadata.
* @returns The {@link Metadata} associated with the logger.
*/
getMetadata() {
return structuredClone(this.metadata);
}
/**
* Sets a key-value pair in the metadata. If the key already exists, it will be overwritten.
*
* @param key Key of the metadata to be set.
* May be any string other than 'message', which is reserved for the actual message logged.
* @param value Value of the metadata to be set.
*/
setMetadata(key, value) {
this.metadata[key] = value;
}
/**
* Clears the Logger's metadata.
*/
clearMetadata() {
this.metadata = {};
}
log(logLevel, message, metadata) {
if (LOG_LEVEL_SEVERITY[logLevel] < LOG_LEVEL_SEVERITY[this.level]) {
return;
}
// We need to provide the date to Datadog. Otherwise, the date is set to when they receive the log.
const date = Date.now();
if (message.length > MAX_LOG_MESSAGE_SIZE) {
message = `${message.substring(0, MAX_LOG_MESSAGE_SIZE)}${LOG_LINE_TRUNCATED_SUFFIX}`;
}
let processedMetadata = Logger.snakifyKeys({ ...this.metadata, ...metadata, logMessageSize: message.length });
processedMetadata = Logger.pruneSensitiveMetadata(processedMetadata);
const processedLogs = {
...processedMetadata,
message,
date,
status: logLevel,
};
if (process.env.NODE_ENV === 'development') {
const coloredMessage = Logger.colorize(message, processedLogs, logLevel);
const metadata = {
date: new Date(processedLogs.date).toISOString(),
...(processedMetadata.error && { error: processedMetadata.error }),
};
const metadataString = Object.keys(metadata).length > 1 ? ` ${JSON.stringify(metadata, null, 2)}` : ` ${JSON.stringify(metadata)}`;
console[logLevel](`${coloredMessage}${metadataString}`);
}
else {
console[logLevel](JSON.stringify(processedLogs));
}
}
static snakifyKeys(value) {
const result = {};
for (const key in value) {
let deepValue;
if (Array.isArray(value[key])) {
if (value[key].every(v => typeof v === 'object')) {
deepValue = value[key].map(item => this.snakifyKeys(item));
}
else {
deepValue = value[key];
}
}
else if (typeof value[key] === 'object' && value[key] !== null) {
deepValue = this.snakifyKeys(value[key]);
}
else {
deepValue = value[key];
}
const snakifiedKey = key.replace(/[\w](?<!_)([A-Z])/g, k => `${k[0]}_${k[1]}`).toLowerCase();
result[snakifiedKey] = deepValue;
}
return result;
}
static pruneSensitiveMetadata(metadata, topLevelMeta) {
const prunedMetadata = {};
for (const key in metadata) {
if (LOGMETA_BLACKLIST.includes(key)) {
prunedMetadata[key] = '[REDACTED]';
(topLevelMeta ?? prunedMetadata).has_sensitive_attribute = true;
}
else if (Array.isArray(metadata[key])) {
if (metadata[key].every(value => typeof value === 'object')) {
prunedMetadata[key] = metadata[key].map(value => Logger.pruneSensitiveMetadata(value, topLevelMeta ?? prunedMetadata));
}
else {
prunedMetadata[key] = metadata[key];
}
}
else if (typeof metadata[key] === 'object' && metadata[key] !== null) {
prunedMetadata[key] = Logger.pruneSensitiveMetadata(metadata[key], topLevelMeta ?? prunedMetadata);
}
else {
prunedMetadata[key] = metadata[key];
}
}
return prunedMetadata;
}
/**
* Colorizes the log message based on the log level and status codes.
* @param message The message to colorize.
* @param metadata The metadata associated with the log.
* @param logLevel The log level of the message.
* @returns The colorized output string.
*/
static colorize(message, metadata, logLevel) {
if (!process.stdout.isTTY) {
return `${logLevel}: ${message}`;
}
// Extract status code from logs
let statusCode;
if (metadata.http && typeof metadata.http === 'object' && !Array.isArray(metadata.http)) {
const statusCodeValue = metadata.http.status_code;
if (typeof statusCodeValue === 'number') {
statusCode = statusCodeValue;
}
else if (typeof statusCodeValue === 'string') {
statusCode = parseInt(statusCodeValue, 10);
}
}
// Color based on status code first
if (statusCode) {
if (statusCode >= 400) {
return `${util.styleText('red', logLevel)}: ${util.styleText('red', message)}`;
}
else if (statusCode >= 300) {
return `${util.styleText('yellow', logLevel)}: ${util.styleText('yellow', message)}`;
}
else if (statusCode >= 200) {
return `${util.styleText('green', logLevel)}: ${util.styleText('green', message)}`;
}
}
// Fall back to log level if no status code found
switch (logLevel) {
case 'error':
return `${util.styleText('red', logLevel)}: ${util.styleText('red', message)}`;
case 'warn':
return `${util.styleText('yellow', logLevel)}: ${util.styleText('yellow', message)}`;
case 'info':
return `${util.styleText('green', logLevel)}: ${util.styleText('green', message)}`;
case 'debug':
return `${util.styleText('cyan', logLevel)}: ${util.styleText('cyan', message)}`;
default:
return `${logLevel}: ${message}`;
}
}
}
const NULL_LOGGER = new Logger();
NULL_LOGGER.setLevel('disabled');
/**
* Initializes a Cache backed by the Redis instance at the provided url if present, or a LocalCache otherwise.
*
* @param redisUrl - The redis url to connect to (optional).
* @returns A cache instance.
*/
function create(redisUrl) {
const cacheInstance = redisUrl ? new cachette.RedisCache(redisUrl) : new cachette.LocalCache();
// Intended: the correlation id will be the same for all logs of Cachette.
const correlationId = crypto.randomUUID();
const logger = new Logger({ correlation_id: correlationId });
cacheInstance
.on('info', message => {
logger.info(message);
})
.on('warn', message => {
logger.warn(message);
})
.on('error', message => {
logger.error(message);
});
return cacheInstance;
}
const Cache = { create };
if (process.env.NODE_ENV === 'production' && process.env.DD_TRACE_ENABLED === 'true') {
// List of options available to the tracer: https://datadoghq.dev/dd-trace-js/interfaces/export_.TracerOptions.html
const apmConfig = { logInjection: false, profiling: false, sampleRate: 0 };
if (process.env.DD_APM_ENABLED == 'true') {
apmConfig.logInjection = true;
apmConfig.sampleRate = 1;
}
else {
// The "enable tracing" boolean is not exposed as part of the TracerOptions so we
// have to do so through env vars. Setting here instead of in services' config
// to avoid too many env vars to "flip" when enabling / disabling APM.
process.env.DD_TRACING_ENABLED = 'false';
}
// Profiling is extra $$$ so we're setting it as an "extra" when needed
if (process.env.DD_APM_PROFILING_ENABLED == 'true') {
apmConfig.profiling = true;
}
const tags = {};
// Using DD_TRACE_AGENT_URL as a hack to know if we're running inside of Kubernetes
// since this variable is not needed in Elastic Beanstalk of Lambda
if (process.env.DD_TRACE_AGENT_URL) {
tags.pod_name = process.env.HOSTNAME;
}
tracer.init({
...apmConfig, // Conditionally enable APM tracing & profiling
runtimeMetrics: true, // Always enable runtime metrics https://docs.datadoghq.com/tracing/metrics/runtime_metrics/nodejs/?tab=environmentvariables
tags,
}); // initialized in a different file to avoid hoisting.
}
/**
* WARNING to projects importing this
* Even if dd-tracer documents that instrumentation happens at **.init() time**
* (see https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/nodejs/#import-and-initialize-the-tracer ),
* 1. dd-tracer has no choice but to do some stuff **at import time**
* 2. dd-tracer is not exempt from bugs, like https://github.com/DataDog/dd-trace-js/issues/5211
*
* So, do not assume that dd-trace is *fully off* by default! Some of it runs!
* If you want to entirely disable all of dd-trace (e.g. during unit tests),
* then you must set `DD_TRACE_ENABLED=false`
*
* Q: Why not move the dd-trace import inside the `if DD_TRACE_ENABLED` condition, then?
* A: Because the future is ESM, and ESM imports must be static and top-level.
*/
const DD_TRACER = tracer;
/**
* Error class meant to be returned by integrations in case of exceptions. These errors will be caught and handled
* appropriately. Any other error would result in an unhandled server error accompanied by a 500 status code.
*
* @field message - The error message
* @field status - The HTTP status code to return
* @field retryAfter - The minimum number of seconds to wait before retrying the request.
* @field responseHeaders - The raw response headers from the provider, when the error came from a provider response.
*/
class HttpError extends Error {
status;
// Not readonly: the SDK populates these centrally in Provider.handleError after the error is built/returned.
retryAfter = undefined;
responseHeaders = undefined;
constructor(message, status, options = {}) {
super(message);
this.status = status;
this.name = this.constructor.name;
this.retryAfter = options.retryAfter;
this.responseHeaders = options.responseHeaders;
}
}
/**
* Used to generate a 400 Bad Request. Usually used when something is missing to properly handle the request.
*/
class BadRequestError extends HttpError {
constructor(message) {
super(message || 'Bad request', 400);
}
}
/**
* Used to generate a 401 Unauthorized. Usually used when the credentials are missing or invalid.
*/
class UnauthorizedError extends HttpError {
constructor(message) {
super(message || 'Unauthorized', 401);
}
}
/**
* Used to generate a 403 Forbidden. Usually used when user lacks sufficient permission to access a ressource.
*/
class ForbiddenError extends HttpError {
constructor(message) {
super(message || 'Forbidden', 403);
}
}
/**
* Used to generate a 404 Not Found. Usually used when the requested `Item` is not found.
*/
class NotFoundError extends HttpError {
constructor(message) {
super(message || 'Not found', 404);
}
}
/**
* Used to generate a 408 Timeout Error. Usually used when the call length exceeds the received Operation Deadline.
*/
class TimeoutError extends HttpError {
constructor(message) {
super(message || 'Not found', 408);
}
}
/**
* Used to generate a 410 Resource Gone.
*
* @deprecated Use ProviderInstanceLockedError instead when the provider instance is locked or unavailable.
*/
class ResourceGoneError extends HttpError {
constructor(message) {
super(message || 'Resource Gone', 410);
}
}
/**
* Used to generate a 422 Unprocessable Entity. Usually used when an operation is invalid.
*/
class UnprocessableEntityError extends HttpError {
constructor(message) {
super(message || 'Unprocessable Entity', 422);
}
}
/**
* Used to generate a 423 Provider Instance Locked.
*/
class ProviderInstanceLockedError extends HttpError {
constructor(message, options = {}) {
super(message || 'Provider instance locked or unavailable', 423, options);
}
}
/**
* Used to generate a 429 Rate Limit Exceeded. Usually used when an operation triggers or would trigger a rate limit
* error on the provider's side.
*/
class RateLimitExceededError extends HttpError {
constructor(message, options = {}) {
super(message || 'Rate Limit Exceeded', 429, options);
}
}
var httpErrors = /*#__PURE__*/Object.freeze({
__proto__: null,
BadRequestError: BadRequestError,
ForbiddenError: ForbiddenError,
HttpError: HttpError,
NotFoundError: NotFoundError,
ProviderInstanceLockedError: ProviderInstanceLockedError,
RateLimitExceededError: RateLimitExceededError,
ResourceGoneError: ResourceGoneError,
TimeoutError: TimeoutError,
UnauthorizedError: UnauthorizedError,
UnprocessableEntityError: UnprocessableEntityError
});
class InvalidHandler extends Error {
}
/**
* Parses a Retry-After header into a non-negative integer number of seconds to wait. Handles both the
* delay-seconds and HTTP-date forms defined by RFC 9110, and rounds up so callers never retry earlier than
* the provider asked. Returns undefined when the header is absent, blank, or unparseable.
*
* Internal to the SDK (errors.ts is not re-exported from index): the SDK populates HttpError.retryAfter
* centrally in Provider.handleError, so integrations don't need to call this themselves.
*/
function parseRetryAfterSeconds(headerValue) {
const trimmed = headerValue?.trim();
if (!trimmed) {
return undefined;
}
const asSeconds = Number(trimmed);
if (!Number.isNaN(asSeconds)) {
return Math.max(0, Math.ceil(asSeconds));
}
const asDate = new Date(trimmed).valueOf();
if (!Number.isNaN(asDate)) {
return Math.max(0, Math.ceil((asDate - Date.now()) / 1000));
}
return undefined;
}
/**
* Processes provider response codes and returns the corresponding errors to be translated further in our responses
*
* @param responseStatus the reponseStatus of the request. Any HTTP response code passed here will result in an error!
* @param message The message returned by the provider
*/
// Keep in errors.ts instead of httpErrors.ts because we do not need to export it outside of the sdk
function buildHttpError(responseStatus, message) {
let httpError;
if (responseStatus === 400) {
httpError = new BadRequestError(message);
}
else if (responseStatus === 401) {
httpError = new UnauthorizedError(message);
}
else if (responseStatus === 403) {
httpError = new ForbiddenError(message);
}
else if (responseStatus === 404) {
httpError = new NotFoundError(message);
}
else if (responseStatus === 408) {
httpError = new TimeoutError(message);
}
else if (responseStatus === 410) {
httpError = new ResourceGoneError(message);
}
else if (responseStatus === 422) {
httpError = new UnprocessableEntityError(message);
}
else if (responseStatus === 423) {
httpError = new ProviderInstanceLockedError(message);
}
else if (responseStatus === 429) {
httpError = new RateLimitExceededError(message);
}
else {
httpError = new HttpError(message, responseStatus);
}
return httpError;
}
function assertValidPath(path) {
if (!path.startsWith('/')) {
throw new InvalidHandler(`The provided path '${path}' is invalid. All paths must start with a '/'.`);
}
if (path.length > 1 && path.endsWith('/')) {
throw new InvalidHandler(`The provided path '${path}' is invalid. Paths must not end with a '/'.`);
}
}
function assertValidConfiguration(path, pathWithIdentifier, handlers) {
if (path === pathWithIdentifier) {
const individualHandlers = ['getItem', 'updateItem', 'deleteItem'];
const collectionHandlers = ['getCollection', 'createItem'];
const hasIndividualHandlers = individualHandlers.some(handler => handler in handlers);
const hasCollectionHandlers = collectionHandlers.some(handler => handler in handlers);
if (hasIndividualHandlers && hasCollectionHandlers) {
throw new InvalidHandler(`The provided path '${path}' doesn't differentiate between individual and collection level operation, so you cannot define both.`);
}
}
}
function parsePath(path) {
const pathParts = path.split('/');
const lastPart = pathParts.at(-1);
if (pathParts.length > 1 && lastPart && lastPart.startsWith(':')) {
pathParts.pop();
}
return { pathWithIdentifier: path, path: pathParts.join('/') };
}
function assertCreateItemRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid CreateItemRequestPayload');
}
if (Object.keys(body).length === 0) {
throw new UnprocessableEntityError('Empty CreateItemRequestPayload');
}
}
function assertUpdateItemRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid UpdateItemRequestPayload');
}
if (Object.keys(body).length === 0) {
throw new UnprocessableEntityError('Empty UpdateItemRequestPayload');
}
}
function assertWebhookParseRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid WebhookParseRequestPayload');
}
if (!('payload' in body) || typeof body.payload !== 'string') {
throw new BadRequestError("Missing required 'payload' property in WebhookParseRequestPayload");
}
if (!('url' in body) || typeof body.url !== 'string') {
throw new BadRequestError("Missing required 'url' property in WebhookParseRequestPayload");
}
}
function assertRichTextParseRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid RichTextParseRequestPayload');
}
if (!('itemPath' in body) || typeof body.itemPath !== 'string') {
throw new BadRequestError("Missing required 'itemPath' property in RichTextParseRequestPayload");
}
if (!('relationName' in body) || typeof body.relationName !== 'string') {
throw new BadRequestError("Missing required 'relationName' property in RichTextParseRequestPayload");
}
if (!('fieldName' in body) || typeof body.fieldName !== 'string') {
throw new BadRequestError("Missing required 'fieldName' property in RichTextParseRequestPayload");
}
if (!('value' in body) || typeof body.value !== 'string') {
throw new BadRequestError("Missing required 'value' property in RichTextParseRequestPayload");
}
}
function assertRichTextDumpRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid RichTextDumpRequestPayload');
}
if (!('itemPath' in body) || typeof body.itemPath !== 'string') {
throw new BadRequestError("Missing required 'itemPath' property in RichTextDumpRequestPayload");
}
if (!('relationName' in body) || typeof body.relationName !== 'string') {
throw new BadRequestError("Missing required 'relationName' property in RichTextDumpRequestPayload");
}
if (!('fieldName' in body) || typeof body.fieldName !== 'string') {
throw new BadRequestError("Missing required 'fieldName' property in RichTextDumpRequestPayload");
}
if (!('root' in body) || typeof body.root !== 'object' || body.root === null) {
throw new BadRequestError("Missing required 'root' property in RichTextDumpRequestPayload");
}
}
function assertWebhookSubscriptionRequestPayload(body) {
if (typeof body !== 'object' || body === null) {
throw new BadRequestError('Invalid WebhookSubscriptionRequestPayload');
}
if (!('itemPath' in body) || typeof body.itemPath !== 'string') {
throw new BadRequestError("Missing required 'itemPath' property in WebhookSubscriptionRequestPayload");
}
if (!('targetUrl' in body) || typeof body.targetUrl !== 'string') {
throw new BadRequestError("Missing required 'targetUrl' property in WebhookSubscriptionRequestPayload");
}
if (!('action' in body) || typeof body.action !== 'string') {
throw new BadRequestError("Missing required 'action' property in WebhookSubscriptionRequestPayload");
}
if (!['start', 'stop'].includes(body.action)) {
throw new BadRequestError("Invalid value for 'action' property in WebhookSubscriptionRequestPayload");
}
}
class Handler {
path;
pathWithIdentifier;
handlers;
constructor(inputPath, handlers) {
assertValidPath(inputPath);
const { pathWithIdentifier, path } = parsePath(inputPath);
assertValidConfiguration(path, pathWithIdentifier, handlers);
this.pathWithIdentifier = pathWithIdentifier;
this.path = path;
this.handlers = handlers;
}
generate() {
const router = express.Router({ caseSensitive: true });
console.debug(`\x1b[33mMounting handler at path ${this.pathWithIdentifier}`);
if (this.handlers.getCollection) {
const handler = this.handlers.getCollection;
console.debug(` Enabling getCollection at GET ${this.path}`);
router.get(this.path, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
const collection = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
search: res.locals.search,
selects: res.locals.selects,
filters: res.locals.filters,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
relations: res.locals.relations,
});
res.status(200).send(collection);
});
}
if (this.handlers.createItem) {
const handler = this.handlers.createItem;
console.debug(` Enabling createItem at POST ${this.path}`);
router.post(this.path, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
assertCreateItemRequestPayload(req.body);
res.locals.logger.decorate({ fieldNames: Object.keys(req.body) });
const createItemSummary = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
body: req.body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(201).send(createItemSummary);
});
}
if (this.handlers.createBlob) {
const handler = this.handlers.createBlob;
console.debug(` Enabling createBlob at POST ${this.pathWithIdentifier}`);
router.post(this.path, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
/**
* Some of the integrations, servicenow for example,
* will need to add more information to the form data that is being passed to the upload attachment handler.
* This is why we need to use busboy to parse the form data, extract the information about the file and pass it to the handler.
*/
const bb = busboy({ headers: req.headers });
const formFields = {};
bb.on('field', (name, value) => {
formFields[name] = value;
});
bb.on('file', async (_name, file, info) => {
try {
const body = {
file: file,
mimeType: info.mimeType,
encoding: info.encoding,
filename: info.filename,
};
if (formFields.fileSize) {
body.fileSize = Number(formFields.fileSize);
}
const createdBlob = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(201).send(createdBlob);
}
catch (error) {
if (error instanceof HttpError) {
res.status(error.status).send(error);
}
else {
res.status(500).send({ message: `Error creating the blob: ${error}` });
}
}
});
bb.on('error', error => {
res.status(500).send({ message: `Error parsing the form data: ${error}` });
});
req.pipe(bb);
});
}
if (this.handlers.getItem) {
const handler = this.handlers.getItem;
console.debug(` Enabling getItem at GET ${this.pathWithIdentifier}`);
router.get(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
const item = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(200).send(item);
});
}
if (this.handlers.updateItem) {
const handler = this.handlers.updateItem;
console.debug(` Enabling updateItem at PATCH ${this.pathWithIdentifier}`);
router.patch(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
assertUpdateItemRequestPayload(req.body);
res.locals.logger.decorate({
fieldNames: Object.keys(req.body).filter(k => k !== '__meta'),
});
const item = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
body: req.body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(200).send(item);
});
}
if (this.handlers.deleteItem) {
const handler = this.handlers.deleteItem;
console.debug(` Enabling deleteItem at DELETE ${this.pathWithIdentifier}`);
router.delete(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(204).send(null);
});
}
if (this.handlers.getBlob) {
console.debug(` Enabling getBlob at GET ${this.pathWithIdentifier}`);
const handler = this.handlers.getBlob;
router.get(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
const blob = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.writeHead(200, {
'Content-Type': 'application/octet-stream',
});
const reader = blob.getReader();
let isDone = false;
try {
while (!isDone) {
const chunk = await reader.read();
isDone = chunk.done;
if (chunk.value) {
res.write(chunk.value);
}
}
}
finally {
reader.releaseLock();
}
res.end();
});
}
if (this.handlers.getCredentialAccount) {
const handler = this.handlers.getCredentialAccount;
console.debug(` Enabling getCredentialAccount at GET ${this.pathWithIdentifier}`);
router.get(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
const credentialAccount = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(200).send(credentialAccount);
});
}
if (this.handlers.acknowledgeWebhooks) {
const handler = this.handlers.acknowledgeWebhooks;
console.debug(` Enabling acknowledgeWebhooks at POST ${this.pathWithIdentifier}`);
router.post(this.pathWithIdentifier, async (req, res) => {
assertWebhookParseRequestPayload(req.body);
const response = await handler({
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
body: req.body,
});
res.status(200).send(response);
});
}
if (this.handlers.parseWebhooks) {
const handler = this.handlers.parseWebhooks;
console.debug(` Enabling parseWebhooks at POST ${this.pathWithIdentifier}`);
router.post(this.pathWithIdentifier, async (req, res) => {
assertWebhookParseRequestPayload(req.body);
const response = await handler({
secrets: res.locals.secrets,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
body: req.body,
});
res.status(200).send(response);
});
}
if (this.handlers.updateWebhookSubscriptions) {
const handler = this.handlers.updateWebhookSubscriptions;
console.debug(` Enabling updateWebhookSubscriptions at PUT ${this.pathWithIdentifier}`);
router.put(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
assertWebhookSubscriptionRequestPayload(req.body);
const response = await handler({
secrets: res.locals.secrets,
credentials: res.locals.credentials,
body: req.body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(204).send(response);
});
}
if (this.handlers.parseRichText) {
const handler = this.handlers.parseRichText;
console.debug(` Enabling parseRichText at POST ${this.pathWithIdentifier}`);
router.post(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
assertRichTextParseRequestPayload(req.body);
const response = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
body: req.body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(200).send(response);
});
}
if (this.handlers.dumpRichText) {
const handler = this.handlers.dumpRichText;
console.debug(` Enabling dumpRichText at POST ${this.pathWithIdentifier}`);
router.post(this.pathWithIdentifier, async (req, res) => {
if (!res.locals.credentials) {
throw new UnauthorizedError();
}
assertRichTextDumpRequestPayload(req.body);
const response = await handler({
credentials: res.locals.credentials,
secrets: res.locals.secrets,
body: req.body,
logger: res.locals.logger,
deadline: res.locals.deadline,
requestMetrics: res.locals.requestMetrics,
params: req.params,
query: req.query,
});
res.status(200).send(response);
});
}
console.debug(`\x1b[0m`);
return router;
}
}
function extractCorrelationId(req, res, next) {
res.locals.correlationId = req.header('X-Unito-Correlation-Id') ?? crypto.randomUUID();
next();
}
const CREDENTIALS_HEADER = 'X-Unito-Credentials';
function extractCredentials(req, res, next) {
const credentialsHeader = req.header(CREDENTIALS_HEADER);
if (credentialsHeader) {
let credentials;
try {
credentials = JSON.parse(Buffer.from(credentialsHeader, 'base64').toString('utf8'));
}
catch {
throw new BadRequestError(`Malformed HTTP header ${CREDENTIALS_HEADER}`);
}
res.locals.credentials = credentials;
if (credentials.unitoCredentialId) {
res.locals.logger?.decorate({ credentialId: credentials.unitoCredentialId });
}
if (credentials.unitoUserId) {
res.locals.logger?.decorate({ userId: credentials.unitoUserId });
}
}
next();
}
function onError(err, _req, res, next) {
if (res.headersSent) {
return next(err);
}
let error;
if (err instanceof HttpError) {
if (err.retryAfter) {
res.setHeader('Retry-After', String(err.retryAfter));
}
error = {
code: err.status.toString(),
message: err.message,
details: {
stack: err.stack,
},
};
}
else {
error = {
code: '500',
message: 'Oops! Something went wrong',
originalError: {
code: err.name,
message: err.message,
details: {
stack: err.stack,
},
},
};
}
res.locals.error = structuredClone(error);
// Keep the stack details in development for the Debugger
if (process.env.NODE_ENV !== 'development') {
delete error.details;
delete error.originalError?.details;
}
res.status(Number(error.code)).json(error);
}
// The operators are ordered by their symbol length, in descending order.
// This is necessary because the symbol of an operator can be
// a subset of the symbol of another operator.
//
// For example, the symbol "=" (EQUAL) is a subset of the symbol "!=" (NOT_EQUAL).
const ORDERED_OPERATORS = Object.values(integrationApi.OperatorTypes).sort((o1, o2) => o2.length - o1.length);
function extractFilters(req, res, next) {
const rawFilters = req.query.filter;
res.locals.filters = [];
if (typeof rawFilters === 'string') {
for (const rawFilter of rawFilters.split(',')) {
// Find the operator that appears earliest in the string.
// When two operators start at the same index (e.g. ">=" and ">" both at position 5),
// the longer one wins because ORDERED_OPERATORS is sorted longest-first.
let bestIdx = -1;
let bestOperator;
for (const operator of ORDERED_OPERATORS) {
const idx = rawFilter.indexOf(operator);
if (idx !== -1 && (bestIdx === -1 || idx < bestIdx)) {
bestIdx = idx;
bestOperator = operator;
}
}
if (bestIdx !== -1 && bestOperator) {
const field = rawFilter.slice(0, bestIdx);
const valuesRaw = rawFilter.slice(bestIdx + bestOperator.length);
const values = valuesRaw ? valuesRaw.split('|').map(decodeURIComponent) : [];
res.locals.filters.push({ field, operator: bestOperator, values });
}
}
}
next();
}
function onFinish(req, res, next) {
res.on('finish', function () {
const logger = res.locals.logger ?? new Logger();
const error = res.locals.error;
const endMetrics = res.locals.requestMetrics.endRequest();
const durationInNs = Number(endMetrics.durationNs);
const durationInMs = (durationInNs / 1_000_000) | 0;
const routePath = req.route?.path;
const urlCategory = typeof routePath === 'string' ? `${req.baseUrl}${routePath}` : 'unmatched';
const message = `${req.method} ${req.originalUrl} ${res.statusCode} - ${durationInMs} ms`;
const metadata = {
duration: durationInNs,
externalApiCount: endMetrics.apiCallCount,
externalApiTotalDuration: endMetrics.totalApiDurationNs,
throttleDelayTotal: endMetrics.totalThrottleDelayNs,
// Use reserved and standard attributes of Datadog
// https://app.datadoghq.com/logs/pipelines/standard-attributes
http: {
method: req.method,
status_code: res.statusCode,
url_category: urlCategory,
url_details: { path: req.originalUrl },
},
...(error
? {
error: {
kind: error.message,
stack: (error.originalError?.details?.stack ?? error.details?.stack),
message: error.originalError?.message ?? error.message,
},
}
: {}),
};
if ([404, 429].includes(res.statusCode)) {
logger.warn(message, metadata);
}
else if (res.statusCode >= 400) {
logger.error(message, metadata);
}
else if (req.originalUrl !== '/health') {
// Don't log successful health check requests
logger.info(message, metadata);
}
});
next();
}
function notFound(req, res, _next) {
const error = {
code: '404',
message: `Path ${req.path} not found.`,
};
res.status(404).json(error);
}
const ADDITIONAL_CONTEXT_HEADER = 'X-Unito-Additional-Logging-Context';
const DEBUG_HEADER = 'X-Unito-Debug';
function injectLogger(req, res, next) {
const logger = new Logger({ correlation_id: res.locals.correlationId, request_id: res.locals.requestId });
res.locals.logger = logger;
if (req.header(DEBUG_HEADER)) {
logger.setLevel('debug');
}
const rawAdditionalContext = req.header(ADDITIONAL_CONTEXT_HEADER);
if (typeof rawAdditionalContext === 'string') {
try {
const additionalContext = JSON.parse(rawAdditionalContext);
logger.decorate(additionalContext);
}
catch (error) {
logger.warn(`Failed parsing header ${ADDITIONAL_CONTEXT_HEADER}: ${rawAdditionalContext}`);
}
}
next();
}
/**
* Accumulates per-request metrics for Provider API calls.
*
* Created once per incoming request (in the start middleware), threaded through Context and RequestOptions,
* incremented by Provider on each API call, and logged by the finish middleware.
*/
class RequestMetrics {
_apiCallCount = 0;
_totalApiDurationNs = 0;
_totalThrottleDelayNs = 0;
_requestStartTime;
constructor() {
this._requestStartTime = process.hrtime.bigint();
}
static startRequest() {
return new RequestMetrics();
}
endRequest() {
return {
durationNs: process.hrtime.bigint() - this._requestStartTime,
apiCallCount: this._apiCallCount,
totalApiDurationNs: this._totalApiDurationNs,
totalThrottleDelayNs: this._totalThrottleDelayNs,
};
}
/**
* Record a completed API call.
* Increments the call counter and accumulates the call's network duration and pre-flight
* throttle delay (time spent waiting in the rate limiter before the request was dispatched).
*/
recordExternalApiCall(duration, throttleDelay) {
this._apiCallCount++;
this._totalApiDurationNs += duration;
this._totalThrottleDelayNs += throttleDelay;
}
}
function start(_, res, next) {
res.locals.requestMetrics = RequestMetrics.startRequest();
res.locals.requestId = crypto.randomUUID();
next();
}
const SECRETS_HEADER = 'X-Unito-Secrets';
function extractSecrets(req, res, next) {
const secretsHeader = req.header(SECRETS_HEADER);
if (secretsHeader) {
let secrets;
try {
secrets = JSON.parse(Buffer.from(secretsHeader, 'base64').toString('utf8'));
}
catch {
throw new BadRequestError(`Malformed HTTP header ${SECRETS_HEADER}`);
}
res.locals.secrets = secrets;
}
next();
}
function extractSearch(req, res, next) {
const rawSearch = req.query.search;
if (typeof rawSearch === 'string') {
res.locals.search = rawSearch;
}
else {
res.locals.search = null;
}
next();
}
function extractSelects(req, res, next) {
const rawSelect = req.query.select;
if (typeof rawSelect === 'string') {
res.locals.selects = rawSelect.split(',').map(select => decodeURIComponent(select));
}
else {
res.locals.selects = [];
}
next();
}
function extractRelations(req, res, next) {
const rawRelations = req.query.relations;
res.locals.relations = typeof rawRelations === 'string' ? rawRelations.split(',') : [];
next();
}
class OperationDeadline {
/** Absolute deadline, epoch ms — or Infinity when no header was sent. */
deadlineTimestamp;
/** The AbortSignal that fires at the deadline. Pass THIS to fetch/provider calls. */
signal;
/** @param header deadline timestamp in milliseconds from Epoch, or undefined */
constructor(deadlineInMs) {
if (!deadlineInMs) {
this.deadlineTimestamp = Infinity;
this.signal = new AbortController().signal; // never fires
return;
}
this.deadlineTimestamp = deadlineInMs;
const remaining = this.deadlineTimestamp - Date.now();
if (remaining <= 0) {
throw new TimeoutError('Request already timed out upon reception');
}
this.signal = AbortSignal.timeout(remaining);
}
/** @param header raw X-Unito-Operation-Deadline value (Unix seconds), or undefined */
static fromH