apify-client
Version:
Apify API client for JavaScript
247 lines • 9.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PaginationIterator = void 0;
exports.pluckData = pluckData;
exports.catchNotFoundOrThrow = catchNotFoundOrThrow;
exports.parseDateFields = parseDateFields;
exports.stringifyWebhooksToBase64 = stringifyWebhooksToBase64;
exports.maybeGzipValue = maybeGzipValue;
exports.sliceArrayByByteLength = sliceArrayByByteLength;
exports.isNode = isNode;
exports.isBuffer = isBuffer;
exports.isStream = isStream;
exports.getVersionData = getVersionData;
exports.cast = cast;
exports.asArray = asArray;
exports.applyQueryParamsToUrl = applyQueryParamsToUrl;
const tslib_1 = require("tslib");
const ow_1 = tslib_1.__importDefault(require("ow"));
const NOT_FOUND_STATUS_CODE = 404;
const RECORD_NOT_FOUND_TYPE = 'record-not-found';
const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found';
const MIN_GZIP_BYTES = 1024;
/**
* Returns object's 'data' property or throws if parameter is not an object,
* or an object without a 'data' property.
*/
function pluckData(obj) {
if (typeof obj === 'object' && obj) {
if (typeof obj.data !== 'undefined')
return obj.data;
}
throw new Error(`Expected response object with a "data" property, but received: ${obj}`);
}
/**
* If given HTTP error has NOT_FOUND_STATUS_CODE status code then returns undefined.
* Otherwise rethrows error.
*/
function catchNotFoundOrThrow(err) {
const isNotFoundStatus = err.statusCode === NOT_FOUND_STATUS_CODE;
const isNotFoundMessage = err.type === RECORD_NOT_FOUND_TYPE || err.type === RECORD_OR_TOKEN_NOT_FOUND_TYPE || err.httpMethod === 'head';
const isNotFoundError = isNotFoundStatus && isNotFoundMessage;
if (!isNotFoundError)
throw err;
}
/**
* Traverses JSON structure and converts fields that end with "At" to a Date object (fields such as "modifiedAt" or
* "createdAt").
*
* If you want parse other fields as well, you can provide a custom matcher function shouldParseField(). This
* admittedly awkward approach allows this function to be reused for various purposes without introducing potential
* breaking changes.
*
* If the field cannot be converted to Date, it is left as is.
*/
function parseDateFields(input, shouldParseField = null, depth = 0) {
// Don't go too deep to avoid stack overflows (especially if there is a circular reference). The depth of 3
// corresponds to obj.data.someArrayField.[x].field and should be generally enough.
// TODO: Consider removing this limitation. It might came across as an annoying surprise as it's not communicated.
if (depth > 3) {
return input;
}
if (Array.isArray(input))
return input.map((child) => parseDateFields(child, shouldParseField, depth + 1));
if (!input || typeof input !== 'object')
return input;
return Object.entries(input).reduce((output, [k, v]) => {
const isValObject = !!v && typeof v === 'object';
if (k.endsWith('At') || (shouldParseField && shouldParseField(k))) {
if (v) {
const d = new Date(v);
output[k] = Number.isNaN(d.getTime()) ? v : d;
}
else {
output[k] = v;
}
}
else if (isValObject || Array.isArray(v)) {
output[k] = parseDateFields(v, shouldParseField, depth + 1);
}
else {
output[k] = v;
}
return output;
}, {});
}
/**
* Helper function that converts array of webhooks to base64 string
*/
function stringifyWebhooksToBase64(webhooks) {
if (!webhooks)
return;
const webhooksJson = JSON.stringify(webhooks);
if (isNode()) {
return Buffer.from(webhooksJson, 'utf8').toString('base64');
}
const encoder = new TextEncoder();
const uint8Array = encoder.encode(webhooksJson);
return btoa(String.fromCharCode(...uint8Array));
}
let gzipPromisified;
/**
* Gzip provided value, otherwise returns undefined.
*/
async function maybeGzipValue(value) {
if (!isNode())
return;
if (typeof value !== 'string' && !Buffer.isBuffer(value))
return;
// Request compression is not that important so let's
// skip it instead of throwing for unsupported types.
const areDataLargeEnough = Buffer.byteLength(value) >= MIN_GZIP_BYTES;
if (areDataLargeEnough) {
if (!gzipPromisified) {
const { promisify } = await Promise.resolve().then(() => tslib_1.__importStar(require('node:util')));
const { gzip } = await Promise.resolve().then(() => tslib_1.__importStar(require('node:zlib')));
gzipPromisified = promisify(gzip);
}
return gzipPromisified(value);
}
return undefined;
}
/**
* Helper function slice the items from array to fit the max byte length.
*/
function sliceArrayByByteLength(array, maxByteLength, startIndex) {
const stringByteLength = (str) => (isNode() ? Buffer.byteLength(str) : new Blob([str]).size);
const arrayByteLength = stringByteLength(JSON.stringify(array));
if (arrayByteLength < maxByteLength)
return array;
const slicedArray = [];
let byteLength = 2; // 2 bytes for the empty array []
for (let i = 0; i < array.length; i++) {
const item = array[i];
const itemByteSize = stringByteLength(JSON.stringify(item));
if (itemByteSize > maxByteLength) {
throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${startIndex + i} ` +
`exceeds the maximum allowed size (${maxByteLength} bytes).`);
}
if (byteLength + itemByteSize >= maxByteLength)
break;
byteLength += itemByteSize;
slicedArray.push(item);
}
return slicedArray;
}
function isNode() {
return !!(typeof process !== 'undefined' && process.versions && process.versions.node);
}
function isBuffer(value) {
return ow_1.default.isValid(value, ow_1.default.any(ow_1.default.buffer, ow_1.default.arrayBuffer, ow_1.default.typedArray));
}
function isStream(value) {
return ow_1.default.isValid(value, ow_1.default.object.hasKeys('on', 'pipe'));
}
function getVersionData() {
if (typeof BROWSER_BUILD !== 'undefined') {
return { version: VERSION };
}
// eslint-disable-next-line
return require('../package.json');
}
/**
* Helper class to create async iterators from paginated list endpoints with exclusive start key.
*/
class PaginationIterator {
constructor(options) {
Object.defineProperty(this, "maxPageLimit", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "getPage", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "limit", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "exclusiveStartId", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
this.maxPageLimit = options.maxPageLimit;
this.limit = options.limit;
this.exclusiveStartId = options.exclusiveStartId;
this.getPage = options.getPage;
}
async *[Symbol.asyncIterator]() {
let nextPageExclusiveStartId;
let iterateItemCount = 0;
while (true) {
const pageLimit = this.limit
? Math.min(this.maxPageLimit, this.limit - iterateItemCount)
: this.maxPageLimit;
const pageExclusiveStartId = nextPageExclusiveStartId || this.exclusiveStartId;
const page = await this.getPage({
limit: pageLimit,
exclusiveStartId: pageExclusiveStartId,
});
// There are no more pages to iterate
if (page.items.length === 0)
return;
yield page;
iterateItemCount += page.items.length;
// Limit reached stopping to iterate
if (this.limit && iterateItemCount >= this.limit)
return;
nextPageExclusiveStartId = page.items[page.items.length - 1].id;
}
}
}
exports.PaginationIterator = PaginationIterator;
function cast(input) {
return input;
}
function asArray(value) {
if (Array.isArray(value)) {
return value;
}
return [value];
}
/**
* Adds query parameters to a given URL based on the provided options object.
*/
function applyQueryParamsToUrl(url, options) {
for (const [key, value] of Object.entries(options !== null && options !== void 0 ? options : {})) {
// skip undefined values
if (value === undefined)
continue;
// join array values with a comma
if (Array.isArray(value)) {
url.searchParams.set(key, value.join(','));
continue;
}
url.searchParams.set(key, String(value));
}
return url;
}
//# sourceMappingURL=utils.js.map