@apify/utilities/cjs/index.d.ts

Tools and constants shared across Apify projects.

import { Log } from '@apify/log';
import { Transform, TransformCallback, Readable, PassThrough } from 'node:stream';
import { KeyObject } from 'node:crypto';

/*!
 * This module contains various server utility and helper functions.
 * Note that it automatically exports functions from utilities.client.js
 *
 * Author: Jan Curn (jan@apify.com)
 * Copyright(c) 2015 Apify. All rights reserved.
 *
 */

/**
 * Generates a random cryptographically strong string consisting of 17 alphanumeric characters.
 * This string is similar to MongoDB ObjectIds generated by Meteor.
 */
declare function cryptoRandomObjectId(length?: number): string;
/**
 * Generates unique, deterministic record ID from the provided key with given length (defaults to 17).
 */
declare function deterministicUniqueId(key: string, length?: number): string;
/**
 * Returns a random integer between 0 and max (excluded, unless it is also 0).
 * @param {number} maxExcluded
 * @returns {number}
 */
declare function getRandomInt(maxExcluded: number): number;
/**
 * If 'date' is a String, this function converts and returns it as a Date object.
 * Otherwise, the function returns the original 'date' argument.
 * This function is useful to convert dates transfered via JSON which doesn't natively support dates.
 */
declare function parseDateFromJson(date: string | Date): Date;
/**
 * Returns a Promise object that will wait a specific number of milliseconds.
 * @param {number} millis Time to wait. If the value is not larger than zero, the promise resolves immediately.
 */
declare function delayPromise(millis: number): Promise<void>;
/**
 * Removes an element from an array.
 */
declare function removeFromArray<T>(array: T[], element: T): boolean;
interface RequestLike {
    url: string;
}
interface ResponseLike {
    status: (code: number) => void;
    send: (payload: string) => void;
    headersSent?: boolean;
}
/**
 * A default route for HTTP 404 error page for API endpoints.
 */
declare function http404Route(req: RequestLike, res: ResponseLike): void;
/**
 * Default error handler of Express API endpoints.
 */
declare function expressErrorHandler(err: Error, req: RequestLike, res: ResponseLike, next: (...a: unknown[]) => unknown): void;
type BetterIntervalID = {
    _betterClearInterval: () => void;
};
/**
 * Similar to setInterval() but with two important differences:
 * First, it assumes the function is asynchronous and only schedules its next invocation AFTER the asynchronous function finished.
 * Second, it invokes the function immediately.
 * @param func Function to be periodically executed.
 * For backwards compatibility reasons, it is passed a callback as its first argument during invocation, however that callback has no effect.
 * @param delay The number of milliseconds to wait to next invocation of the function after the current invocation finishes.
 * @returns Object that can be passed to betterClearInterval()
 */
declare function betterSetInterval(func: ((a: (...args: unknown[]) => unknown) => void) | ((...args: unknown[]) => unknown), delay: number): BetterIntervalID;
declare function betterClearInterval(intervalID: BetterIntervalID): void;
/**
 * Escapes a string so that it can be used in regular expression (e.g. converts "myfile.*" to "myfile\\.\\*").
 */
declare function escapeRegExp(str: string): string;
/**
 * String left pad
 */
declare function leftpad(str: string, len: number, ch?: string | number): string;
/**
 * Computes weighted average of 2 values.
 */
declare function weightedAverage(val1: number, weight1: number, val2: number, weight2: number): number;
/**
 * Checks whether username is listed in FORBIDDEN_USERNAMES
 * or matches any root route path.
 */
declare function isForbiddenUsername(username: string): boolean;
/**
 * Executes array of promises in sequence and then returns array where Nth item is result of Nth promise.
 */
declare function sequentializePromises<T>(promises: (Promise<T> | (() => Promise<T>))[]): Promise<T[]>;
/**
 * Helper function for validation if parameter is an instance of given prototype or multiple prototypes.
 */
declare function checkParamPrototypeOrThrow(paramVal: any, paramName: string, prototypes: any, prototypeName: string, isOptional?: boolean): void;
interface Server {
    removeListener(event: string, cb: (...params: any[]) => any): unknown;
    on(event: string, cb: (...params: any[]) => any): unknown;
    listen(port: number): unknown;
}
/**
 * Starts listening at a port specified in the constructor.
 * Unfortunately server.listen() is not a normal function that fails on error, so we need this trickery.
 * Returns a function that calls `server.listen(port)` and resolves once server starts listening.
 *
 * Usage: `promisifyServerListen(server)(1234)`;
 */
declare function promisifyServerListen<T extends Server>(server: T): (port: number) => Promise<void>;
declare function configureLogger(givenLog: Log, isProduction?: boolean): void;
/**
 * Wraps given promise with timeout.
 */
declare function timeoutPromise<T>(promise: Promise<T>, timeoutMillis: number, errorMessage?: string): Promise<unknown>;
/**
 * Removes the leading ^ and trailing $ from regex
 */
declare function createInjectableRegExp(regex: RegExp): RegExp;

/*!
 * This module contains various client-side utility and helper functions.
 *
 * Author: Jan Curn (jan@apify.com)
 * Copyright(c) 2016 Apify. All rights reserved.
 *
 */
/**
 * Returns true if object equals null or undefined, otherwise returns false.
 */
declare function isNullOrUndefined(obj: unknown): boolean;
declare function isBuffer(obj: any): boolean;
/**
 * Converts Date object to ISO string.
 */
declare function dateToString(date: Date, middleT: boolean): string;
/**
 * Ensures a string is shorter than a specified number of character, and truncates it if not,
 * appending a specific suffix to it.
 * @param str
 * @param maxLength
 * @param [suffix] Suffix to be appended to truncated string. Defaults to "...[truncated]".
 */
declare function truncate(str: string, maxLength: number, suffix?: string): string;
/**
 * Gets ordinal suffix for a number (e.g. "nd" for 2).
 */
declare function getOrdinalSuffix(num: number): string;
interface Uri {
    protocol?: string;
    host?: string;
    path?: string;
    query?: string;
    fragment?: string;
    fragmentKey?: Record<string, unknown>;
}
/**
 * @deprecated use `new URL()` instead
 */
declare function parseUrl(str: string): Uri;
declare function normalizeUrl(url: string, keepFragment?: boolean): string | null;
declare function markedSetNofollowLinks(href: string, title: string, text: string, referrerHostname?: string): string;
declare function markedDecreaseHeadsLevel(text: string, level: number): string;
/**
 * Converts integer version number previously generated by buildNumberToInt() or versionNumberToInt()
 * to string in a form 'MAJOR.MINOR' or 'MAJOR.MINOR.BUILD' in case build number is non-zero.
 */
declare function buildOrVersionNumberIntToStr(int: number): string | null;
/**
 * If a property name is invalid for MongoDB or BSON, the function transforms
 * it to a valid form, which can be (most of the time) reversed back using unescapePropertyName().
 * For a detailed list of transformations, see escapeForBson().
 * @private
 */
declare function escapePropertyName(name: string): string;
/**
 * Reverses a string transformed using escapePropertyName() back to its original form.
 * Note that the reverse transformation might not be 100% correct for certain unlikely-to-occur strings
 * (e.g. string contain null chars).
 * @private
 */
declare function unescapePropertyName(name: string): string;
/**
 * Traverses an object, creates a deep clone if requested and transforms object keys and values using a provided function.
 * The `traverseObject` is recursive, hence if the input object has circular references, the function will run into
 * and infinite recursion and crash the Node.js process.
 * @param obj Object to traverse, it must not contain circular references!
 * @param clone If true, object is not modified but cloned.
 * @param transformFunc Function used to transform the property names na value.
 *  It has the following signature: `(key, value) => [key, value]`.
 *  Beware that the transformed value is only set if it !== old value.
 * @returns {*}
 * @private
 */
declare function traverseObject(obj: Record<string, any>, clone: boolean, transformFunc: (key: string, value: unknown) => [string, unknown]): any[] | Record<string, any>;
/**
 * Transforms an object so that it can be stored to MongoDB or serialized to BSON.
 * It does so by transforming prohibited property names (e.g. names starting with "$",
 * containing "." or null char, equal to "toBSON" or "_bsontype") to equivalent full-width Unicode chars
 * which are normally allowed. To revert this transformation, use unescapeFromBson().
 * @param obj Object to be transformed. It must not contain circular references or any complex types (e.g. Maps, Promises etc.)!
 * @param clone If true, the function transforms a deep clone of the object rather than the original object.
 * @returns {*} Transformed object
 */
declare function escapeForBson(obj: Record<string, any>, clone?: boolean): any[] | Record<string, any>;
/**
 * Reverts a transformation of object property names performed by escapeForBson().
 * Note that the reverse transformation might not be 100% equal to the original object
 * for certain unlikely-to-occur property name (e.g. one contain null chars or full-width Unicode chars).
 * @param obj Object to be transformed. It must not contain circular references or any complex types (e.g. Maps, Promises etc.)!
 * @param clone If true, the function transforms a deep clone of the object rather than the original object.
 * @returns {*} Transformed object.
 */
declare function unescapeFromBson(obj: Record<string, any>, clone?: boolean): Record<string, any>;
/**
 * Determines whether an object contains property names that cannot be stored to MongoDB.
 * See escapeForBson() for more details.
 * Note that this function only works with objects that are serializable to JSON!
 * @param obj Object to be checked. It must not contain circular references or any complex types (e.g. Maps, Promises etc.)!
 * @returns {boolean} Returns true if object is invalid, otherwise it returns false.
 */
declare function isBadForMongo(obj: Record<string, any>): boolean;
declare class JsonVariable {
    readonly name: string;
    constructor(name: string);
    getToken(): string;
}
/**
 * Stringifies provided value to JSON with a difference that supports functions that
 * are stringified using .toString() method.
 *
 * In addition to that supports instances of JsonVariable('my.token') that are replaced
 * with a {{my.token}}.
 */
declare function jsonStringifyExtended(value: Record<string, any>, replacer?: ((k: string, val: unknown) => unknown) | null, space?: number): string;
/**
 * Splits a full name into the first name and last name, trimming all internal and external spaces.
 * Returns an array with two elements or null if splitting is not possible.
 */
declare function splitFullName(fullName: string): (string | null)[];
/**
 * Perform a Regex test on a given URL to see if it is relative.
 */
declare function isUrlRelative(url: string): boolean;

declare class RetryableError extends Error {
    readonly error: Error;
    constructor(error: Error, ...args: unknown[]);
}
interface RetryableError extends Error {
}
declare function retryWithExpBackoff<T>(params?: {
    func?: (...args: unknown[]) => T | Promise<T>;
    expBackoffMillis?: number;
    expBackoffMaxRepeats?: number;
}): Promise<T>;

declare enum CHECK_TYPES {
    MONGODB_PING = "MONGODB_PING",
    MONGODB_READ = "MONGODB_READ",
    MONGODB_WRITE = "MONGODB_WRITE",
    REDIS = "REDIS",// Old alias for 'REDIS_WRITE', deprecated
    REDIS_PING = "REDIS_PING",
    REDIS_WRITE = "REDIS_WRITE"
}
type CheckType<T extends Record<string, any> = Record<string, any>> = {
    client: T;
    type: CHECK_TYPES;
};
interface HealthCheckerOptions {
    checks: CheckType[];
    redisPrefix?: string;
    redisTtlSecs?: number;
    checkTimeoutMillis?: number;
    mongoDbWriteTestCollection?: string;
    mongoDbWriteTestRemoveOlderThanSecs?: number;
}
/**
 * Provides health-checking functionality to ensure that connection to Redis and MongoDB is working.
 *
 * Example use:
 *
 * ```javascript
 * const redis = new Redis();
 * const mongo = await MongoClient.connect('mongodb://127.0.0.1:3001/my-db');
 *
 * const checks = [{
 *     client: redis,
 *     type: HealthChecker.CHECK_TYPES.REDIS,
 * }, {
 *     client: mongo.db('my-db'),
 *     type: HealthChecker.CHECK_TYPES.MONGODB_READ,
 * }];
 *
 * const checker = new HealthChecker({ checks });
 * setInterval(() => checker.ensureIsHealthy().then(() => console.log('ok'), err => console.log(err)), 5000);
 * ```
 */
declare class HealthChecker {
    private readonly options;
    static readonly CHECK_TYPES: typeof CHECK_TYPES;
    checks: CheckType[];
    redisPrefix: string;
    redisTtlSecs: number;
    checkTimeoutMillis: number;
    mongoDbWriteTestCollection: string;
    mongoDbWriteTestRemoveOlderThanSecs: number;
    constructor(options: HealthCheckerOptions);
    ensureIsHealthy(): Promise<void>;
    _validateCheck(check: CheckType): void;
    _performCheck(check: CheckType): Promise<void>;
    _testMongoDbPing({ client }: CheckType): Promise<void>;
    _testMongoDbRead({ client }: CheckType): Promise<void>;
    _testMongoDbWrite({ client }: CheckType): Promise<void>;
    _testRedisPing({ client }: CheckType): Promise<void>;
    _testRedisWrite({ client }: CheckType): Promise<void>;
}

/**
 * A transforming stream which accepts string/Buffer data with JSON Lines objects on input
 * and emits 'object' event for every parsed JavaScript objects.
 * The stream passes through the original data.
 * Each JSON object is expected to be on a separate line, some lines might be empty or contain whitespace.
 * After each JSON object there needs to be '\n' or end of stream.
 * This stream is especially useful for processing stream from Docker engine, such as:
 *
 * <pre>
 *  {"status":"Preparing","progressDetail":{},"id":"e0380bb6c0bb"}
 *  {"status":"Preparing","progressDetail":{},"id":"9f8566ee5135"}
 *  {"errorDetail":{"message":"no basic auth credentials"},"error":"no basic auth credentials"}
 * </pre>
 *
 * **WARNING**: You still need to consume the `data` event from the transformed stream,
 * otherwise the internal buffers will get full and the stream might be corked.
 */
declare class ParseJsonlStream extends Transform {
    private pendingChunk;
    parseLineAndEmitObject(line: string): void;
    _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void;
    _flush(callback: TransformCallback): void;
}

/**
 * Concat data from stream to Buffer
 */
declare function concatStreamToBuffer(stream: Readable | PassThrough): Promise<Buffer>;
/**
 * Flushes the provided stream into a Buffer and transforms
 * it to a String using the provided encoding or utf-8 as default.
 */
declare function readStreamToString(stream: Readable | PassThrough, encoding?: BufferEncoding): Promise<string>;

declare class WebhookPayloadTemplateError extends Error {
    constructor(message?: string);
}
declare class InvalidJsonError extends WebhookPayloadTemplateError {
    constructor(originalError: Error);
}
declare class InvalidVariableError extends Error {
    constructor(variable?: string);
}
/**
 * WebhookPayloadTemplate enables creation and parsing of webhook payload template strings.
 * Template strings are JSON that may include template variables enclosed in double
 * curly brackets: `{{variable}}`. When the template is parsed, variables are replaced
 * with values from a provided context.
 *
 * This is useful to create dynamic webhook payloads where a template is saved on webhook
 * creation and then variables are dynamically added on webhook dispatch.
 *
 * **Example:**
 * ```js
 * const payloadTemplate = `{
 *    "id": "some-id",
 *    "createdAt": "2019-05-08T15:22:21.095Z",
 *    "dataToSend": {{data}}
 * }`
 *
 * const data = {
 *     status: 200,
 *     body: 'hello world'
 * }
 *
 * const payloadObject = WebhookPayloadTemplate.parse(payloadTemplate, null, { data })
 * ```
 *
 * **Produces:**
 * ```js
 * {
 *     id: "some-id",
 *    createdAt: '2019-05-08T15:22:21.095Z',
 *    dataToSend: {
 *        status: 200,
 *        body: 'hello world'
 *    }
 * }
 * ```
 * @hideconstructor
 */
declare class WebhookPayloadTemplate {
    private readonly template;
    private readonly allowedVariables;
    private readonly context;
    private payload;
    readonly replacedVariables: {
        variableName: string;
        replacement: string;
    }[];
    constructor(template: string, allowedVariables?: Set<string> | null, context?: Record<string, any>);
    /**
     * Parse existing webhook payload template string into an object, replacing
     * template variables using the provided context.
     *
     * Parse also validates the template structure, so it can be used
     * to check validity of the template JSON and usage of allowedVariables.
     */
    static parse(payloadTemplate: string, allowedVariables?: Set<string> | null, context?: Record<string, any>, options?: {
        interpolateStrings?: boolean;
    }): Record<string, any>;
    /**
     * Stringify an object into a webhook payload template.
     * Values created using `getTemplateVariable('foo.bar')`
     * will be stringified to `{{foo.bar}}` template variable.
     */
    static stringify(objectTemplate: Record<string, any>, replacer?: ((_: any) => string) | null, indent?: number): string;
    /**
     * Produces an instance of a template variable that can be used
     * in objects and will be stringified into `{{variableName}}` syntax.
     *
     * **Example:**
     * ```js
     * const resourceVariable = WebhookPayloadTemplate.getVariable('resource');
     * const objectTemplate = {
     *     foo: 'foo',
     *     bar: ['bar'],
     *     res: resourceVariable,
     * }
     *
     * const payloadTemplate = WebhookPayloadTemplate.stringify(objectTemplate);
     * ```
     *
     * **Produces:**
     * ```json
     * {
     *     "foo": "foo",
     *     "bar": ["bar"],
     *     "res": {{resource}},
     * }
     * ```
     */
    static getVariable(variableName: string): JsonVariable;
    private _parse;
    private _interpolate;
    private _interpolateString;
    private _interpolateObject;
    private _interpolateArray;
    private _findPositionOfNextVariable;
    private _isVariableInsideString;
    private _countUnescapedDoubleQuotesUpToIndex;
    private _replaceVariable;
    private _validateVariableName;
    private _getVariableValue;
    private _getVariableReplacement;
}

type DecryptOptions = {
    privateKey: KeyObject;
    encryptedPassword: string;
    encryptedValue: string;
};
type EncryptOptions = {
    publicKey: KeyObject;
    value: string;
};
/**
 * It encrypts the given value using AES cipher and the password for encryption using the public key.
 * NOTE: The encryption password is a string of encryption key and initial vector used for cipher.
 * It returns the encrypted password and encrypted value in BASE64 format.
 *
 * @param publicKey {KeyObject} Public key used for encryption
 * @param value {string} Value to be encrypted
 * @returns {Object<encryptedPassword, encryptedValue>}
 */
declare function publicEncrypt({ publicKey, value }: EncryptOptions): {
    encryptedPassword: string;
    encryptedValue: string;
};
/**
 * It decrypts encrypted password using private key
 * and uses the password(consists of encrypted key and initial vector)
 * to decrypt the encrypted value.
 *
 * @param privateKey {KeyObject} Private key used for decryption
 * @param encryptedPassword {string} Password in Base64 encrypted using private key
 * @param encryptedValue {string} Content in Base64 encrypted using AES cipher
 * @returns {string}
 */
declare function privateDecrypt({ privateKey, encryptedPassword, encryptedValue, }: DecryptOptions): string;

/**
 * Extract import statements from the code.
 */
declare function separateImports(code: string): {
    code: string;
    imports: string;
};

declare enum CodeHashMetaKey {
    VERSION = "v",
    USER = "u"
}
/**
 * Allows hashing of an Actor input together with some metadata into a shareable link for the "Run on Apify" button.
 * Uses a common secret for checking the signatures.
 *
 * The hash consists of 3 parts separated by a dot, as in `ABC.DEF.GHI`, each being a base64url encoded string:
 *  - `meta` object with the `version` and `user` properties.
 *  - `data` data object (the one that gets encoded)
 *  - `signature` used for verification of the URL hash, computed from the `meta` and `data` objects
 */
declare class CodeHashManager {
    private readonly secret;
    static readonly SECTION_SEPARATOR = ".";
    static readonly VERSION = 1;
    constructor(secret: string);
    /**
     * Encodes object (e.g. input for actor) to a string hash and uses the `secret` to sign the hash.
     */
    encode<T extends object>(data: T, userId: string): string;
    decode(urlHash: string): {
        data: any;
        meta: {
            userId: any;
            version: any;
            isSignatureValid: boolean;
        };
    };
    private toBase64;
    private fromBase64;
    private generateSignature;
}

/**
 * Generates an HMAC signature and encodes it using Base62.
 * Base62 encoding reduces the signature length.
 *
 * @param secretKey {string} Secret key used for signing signatures
 * @param message {string} Message to be signed
 * @returns string
 */
declare function createHmacSignature(secretKey: string, message: string): string;

/**
 * Creates a secure signature for a resource like a dataset or key-value store.
 * This signature is used to generate a signed URL for authenticated access, which can be expiring or permanent.
 * The signature is created using HMAC with the provided secret key and includes the resource ID, expiration time, and version.
 *
 * Note: expirationMillis is optional. If not provided, the signature will not expire.
 */
declare function createStorageContentSignature({ resourceId, urlSigningSecretKey, expiresInMillis, version, }: {
    resourceId: string;
    urlSigningSecretKey: string;
    expiresInMillis?: number;
    version?: number;
}): string;

export { type BetterIntervalID, CHECK_TYPES, CodeHashManager, CodeHashMetaKey, HealthChecker, InvalidJsonError, InvalidVariableError, JsonVariable, ParseJsonlStream, RetryableError, WebhookPayloadTemplate, betterClearInterval, betterSetInterval, buildOrVersionNumberIntToStr, checkParamPrototypeOrThrow, concatStreamToBuffer, configureLogger, createHmacSignature, createInjectableRegExp, createStorageContentSignature, cryptoRandomObjectId, dateToString, delayPromise, deterministicUniqueId, escapeForBson, escapePropertyName, escapeRegExp, expressErrorHandler, getOrdinalSuffix, getRandomInt, http404Route, isBadForMongo, isBuffer, isForbiddenUsername, isNullOrUndefined, isUrlRelative, jsonStringifyExtended, leftpad, markedDecreaseHeadsLevel, markedSetNofollowLinks, normalizeUrl, parseDateFromJson, parseUrl, privateDecrypt, promisifyServerListen, publicEncrypt, readStreamToString, removeFromArray, retryWithExpBackoff, separateImports, sequentializePromises, splitFullName, timeoutPromise, traverseObject, truncate, unescapeFromBson, unescapePropertyName, weightedAverage };