@naturalcycles/nodejs-lib
Version:
Standard library for Node.js
261 lines (260 loc) • 11 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.getGot = void 0;
const url_1 = require("url");
const js_lib_1 = require("@naturalcycles/js-lib");
const got_1 = require("got");
const __1 = require("..");
/**
* Returns instance of Got with "reasonable defaults":
*
* 1. Error handler hook that prints helpful errors.
* 2. Hooks that log start/end of request (optional, false by default).
* 3. Reasonable defaults(tm), e.g non-infinite Timeout
* 4. Preserves error stack traces (!) (experimental!)
*/
function getGot(opt = {}) {
opt.logger || (opt.logger = console);
if (opt.debug) {
opt.logStart = opt.logFinished = opt.logResponse = opt.logRequest = true;
}
return got_1.default.extend({
// Most-important is to set to anything non-empty (so, requests don't "hang" by default).
// Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
// Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
// before server times out with 503).
//
// UPD 2021-11-27
// There are 2 types/strategies for requests:
// 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
// 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
//
// Here it's hard to set the default timeout right for both use-cases.
// So, if it's important, you should override it according to your use-cases:
// - set it longer for Type 1 (e.g 120 seconds)
// - set it shorter for Type 2 (e.g 10/20 seconds)
// Please beware of default Retry strategy of Got:
// by default it will retry 2 times (after first try)
// First delay between tries will be ~1 second, then ~2 seconds
// Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
// So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
// So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
// Which definitely doesn't fit into default "RequestTimeout"
timeout: 60000,
...opt,
handlers: [
(options, next) => {
options.context = {
...options.context,
started: Date.now(),
// This is to preserve original stack trace
// https://github.com/sindresorhus/got/blob/main/documentation/async-stack-traces.md
err: new Error('RequestError'),
};
return next(options);
},
],
hooks: {
...opt.hooks,
beforeError: [
...(opt.hooks?.beforeError || []),
// User hooks go BEFORE
gotErrorHook(opt),
],
beforeRequest: [
gotBeforeRequestHook(opt),
// User hooks go AFTER
...(opt.hooks?.beforeRequest || []),
],
beforeRetry: [
gotBeforeRetryHook(opt),
// User hooks go AFTER
...(opt.hooks?.beforeRetry || []),
],
afterResponse: [
...(opt.hooks?.afterResponse || []),
// User hooks go BEFORE
gotAfterResponseHook(opt),
],
},
});
}
exports.getGot = getGot;
/**
* Without this hook (default behaviour):
*
* HTTPError: Response code 422 (Unprocessable Entity)
* at EventEmitter.<anonymous> (.../node_modules/got/dist/source/as-promise.js:118:31)
* at processTicksAndRejections (internal/process/task_queues.js:97:5) {
* name: 'HTTPError'
*
*
* With this hook:
*
* HTTPError 422 GET http://a.com/err?q=1 in 8 ms
* {
* message: 'Reference already exists',
* documentation_url: 'https://developer.github.com/v3/git/refs/#create-a-reference'
* }
*
* Features:
* 1. Includes original method and URL (including e.g searchParams) in the error message.
* 2. Includes response.body in the error message (limited length).
* 3. Auto-detects and parses JSON response body (limited length).
* 4. Includes time spent (gotBeforeRequestHook must also be enabled).
* UPD: excluded now to allow automatic Sentry error grouping
*/
function gotErrorHook(opt = {}) {
const { maxResponseLength = 10000 } = opt;
return err => {
const statusCode = err.response?.statusCode || 0;
const { method, url, prefixUrl } = err.options;
const shortUrl = getShortUrl(opt, url, prefixUrl);
const { started, retryCount } = (err.request?.options.context || {});
const body = err.response?.body
? (0, __1.inspectAny)(err.response.body, {
maxLen: maxResponseLength,
colors: false,
})
: err.message;
// We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
opt.logger.log([
' <<',
statusCode,
method,
shortUrl,
retryCount && `(retry ${retryCount})`,
'error',
started && 'in ' + (0, js_lib_1._since)(started),
]
.filter(Boolean)
.join(' '));
// timings are not part of err.message to allow automatic error grouping in Sentry
// Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
.filter(Boolean)
.join('\n');
const stack = err.options.context?.err?.stack;
if (stack) {
const originalStack = err.stack.split('\n');
let originalStackIndex = originalStack.findIndex(line => line.includes(' at '));
if (originalStackIndex === -1)
originalStackIndex = originalStack.length - 1;
// Skipping first line as it has RequestError: ...
// Skipping second line as it's known to be from e.g at got_1.default.extend.handlers
const syntheticStack = stack.split('\n').slice(2);
let firstNonNodeModulesIndex = syntheticStack.findIndex(line => !line.includes('node_modules'));
if (firstNonNodeModulesIndex === -1)
firstNonNodeModulesIndex = 0;
err.stack = [
// First lines of original error
...originalStack.slice(0, originalStackIndex),
// Other lines from "Synthetic error"
...syntheticStack.slice(firstNonNodeModulesIndex),
].join('\n');
// err.stack += '\n --' + stack.replace('Error: RequestError', '')
}
return err;
};
}
function gotBeforeRequestHook(opt) {
return options => {
if (opt.logStart) {
const { retryCount } = options.context;
const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
opt.logger.log([' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '));
}
if (opt.logRequest) {
const body = options.json || options.body;
if (body) {
opt.logger.log(body);
}
}
};
}
// Here we log always, because it's similar to ErrorHook - we always log errors
// Because Retries are always result of some Error
function gotBeforeRetryHook(opt) {
const { maxResponseLength = 10000 } = opt;
return (options, err, retryCount) => {
// opt.logger!.log('beforeRetry', retryCount)
const statusCode = err?.response?.statusCode || 0;
if (statusCode && statusCode < 300) {
// todo: possibly remove the log message completely in the future
// opt.logger!.log(
// `skipping got.beforeRetry hook as statusCode is ${statusCode}, err.msg is ${err?.message}`,
// )
return;
}
const { method, url, prefixUrl } = options;
const shortUrl = getShortUrl(opt, url, prefixUrl);
const { started } = options.context;
Object.assign(options.context, { retryCount });
const body = err?.response?.body
? (0, __1.inspectAny)(err.response.body, {
maxLen: maxResponseLength,
colors: false,
})
: err?.message;
// We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
opt.logger.warn([
[
' <<',
statusCode,
method,
shortUrl,
retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
'error',
started && 'in ' + (0, js_lib_1._since)(started),
]
.filter(Boolean)
.join(' '),
body,
]
.filter(Boolean)
.join('\n'));
};
}
// AfterResponseHook is never called on Error
// So, coloredHttpCode(resp.statusCode) is probably useless
function gotAfterResponseHook(opt = {}) {
return resp => {
const success = resp.statusCode >= 200 && resp.statusCode < 400;
// Errors are not logged here, as they're logged by gotErrorHook
if (opt.logFinished && success) {
const { started, retryCount } = resp.request.options.context;
const { url, prefixUrl, method } = resp.request.options;
const shortUrl = getShortUrl(opt, url, prefixUrl);
opt.logger.log([
' <<',
resp.statusCode,
method,
shortUrl,
retryCount && `(retry ${retryCount - 1})`,
started && 'in ' + (0, js_lib_1._since)(started),
]
.filter(Boolean)
.join(' '));
// console.log(`afterResp! ${resp.request.options.method} ${resp.url}`, { context: resp.request.options.context })
}
// Error responses are not logged, cause they're included in Error message already
if (opt.logResponse && success) {
opt.logger.log((0, __1.inspectAny)(resp.body, { maxLen: opt.maxResponseLength }));
}
return resp;
};
}
function getShortUrl(opt, url, prefixUrl) {
if (url.password) {
url = new url_1.URL(url.toString()); // prevent original url mutation
url.password = '[redacted]';
}
let shortUrl = url.toString();
if (opt.logWithSearchParams === false) {
shortUrl = shortUrl.split('?')[0];
}
if (opt.logWithPrefixUrl === false && prefixUrl && shortUrl.startsWith(prefixUrl)) {
shortUrl = shortUrl.slice(prefixUrl.length);
}
return shortUrl;
}